From 1dd17c92f255c4c29df1287a5481af46587acfed Mon Sep 17 00:00:00 2001
From: Chris Kleeschulte <chrisk@bitpay.com>
Date: Mon, 13 Nov 2017 09:54:02 -0500
Subject: [PATCH 01/15] Bumped version.

---
 package-lock.json | 2 +-
 package.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index e88c1c194..97e4663a1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "bitcore-node",
-  "version": "5.0.0-beta.41",
+  "version": "5.0.0-beta.42",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
diff --git a/package.json b/package.json
index 561b29b31..5efd28928 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
     "node": ">=8.0.0"
   },
   "author": "BitPay <dev@bitpay.com>",
-  "version": "5.0.0-beta.41",
+  "version": "5.0.0-beta.42",
   "main": "./index.js",
   "repository": "git://github.com/bitpay/bitcore-node.git",
   "homepage": "https://github.com/bitpay/bitcore-node",

From 6e20b78b1294edee708cfa129df9b9a3754f2b35 Mon Sep 17 00:00:00 2001
From: Chris Kleeschulte <chrisk@bitpay.com>
Date: Mon, 13 Nov 2017 14:22:15 -0500
Subject: [PATCH 02/15] Bumped version.

---
 package-lock.json | 2 +-
 package.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 97e4663a1..2a2970eb5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "bitcore-node",
-  "version": "5.0.0-beta.42",
+  "version": "5.0.0-beta.43",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
diff --git a/package.json b/package.json
index 5efd28928..18e4cdeb1 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
     "node": ">=8.0.0"
   },
   "author": "BitPay <dev@bitpay.com>",
-  "version": "5.0.0-beta.42",
+  "version": "5.0.0-beta.43",
   "main": "./index.js",
   "repository": "git://github.com/bitpay/bitcore-node.git",
   "homepage": "https://github.com/bitpay/bitcore-node",

From 0e421de8979802f66b1ba5a6b3459b7626964bd4 Mon Sep 17 00:00:00 2001
From: Chris Kleeschulte <chrisk@bitpay.com>
Date: Wed, 22 Nov 2017 15:56:36 -0500
Subject: [PATCH 03/15] Fixed spent status from getAddressHistory.

---
 lib/services/address/index.js     | 2 +-
 lib/services/transaction/index.js | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/services/address/index.js b/lib/services/address/index.js
index f5da7307e..71901e4c7 100644
--- a/lib/services/address/index.js
+++ b/lib/services/address/index.js
@@ -420,7 +420,7 @@ AddressService.prototype._getAddressTxHistory = function(options, callback) {
       });
     }
 
-    self._transaction.getTransaction(id.txid, next);
+    self._transaction.getDetailedTransaction(id.txid, options, next);
 
   }, callback);
 
diff --git a/lib/services/transaction/index.js b/lib/services/transaction/index.js
index e08672024..83bf40a57 100644
--- a/lib/services/transaction/index.js
+++ b/lib/services/transaction/index.js
@@ -16,7 +16,6 @@ function TransactionService(options) {
   this._header = this.node.services.header;
   this._p2p = this.node.services.p2p;
   this._timestamp = this.node.services.timestamp;
-  this._address = this.node.services.address;
   this._network = this.node.network;
 
   if (this._network === 'livenet') {

From 9db5f2bb3408c5ecf677c83a1219747510144710 Mon Sep 17 00:00:00 2001
From: Chris Kleeschulte <chrisk@bitpay.com>
Date: Wed, 22 Nov 2017 16:02:20 -0500
Subject: [PATCH 04/15] Bumped version.

---
 package-lock.json | 2 +-
 package.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 2a2970eb5..d8f28caa0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "bitcore-node",
-  "version": "5.0.0-beta.43",
+  "version": "5.0.0-beta.44",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
diff --git a/package.json b/package.json
index 18e4cdeb1..74dfa8037 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
     "node": ">=8.0.0"
   },
   "author": "BitPay <dev@bitpay.com>",
-  "version": "5.0.0-beta.43",
+  "version": "5.0.0-beta.44",
   "main": "./index.js",
   "repository": "git://github.com/bitpay/bitcore-node.git",
   "homepage": "https://github.com/bitpay/bitcore-node",

From 01a3df31c8ecd8e85a3ea211d4266dd7de0eb117 Mon Sep 17 00:00:00 2001
From: Matias Alejo Garcia <ematiu@gmail.com>
Date: Sun, 11 Feb 2018 15:44:39 -0300
Subject: [PATCH 05/15] fix history sorting when paging results

---
 lib/services/address/index.js       |  19 ++----
 test/services/address/index.unit.js | 101 +++++++++++++++++++++++++++-
 2 files changed, 104 insertions(+), 16 deletions(-)

diff --git a/lib/services/address/index.js b/lib/services/address/index.js
index 71901e4c7..d48d82f13 100644
--- a/lib/services/address/index.js
+++ b/lib/services/address/index.js
@@ -66,7 +66,6 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
   }
 
   async.eachLimit(addresses, 4, function(address, next) {
-
     self._getAddressTxidHistory(address, options, next);
 
   }, function(err) {
@@ -75,21 +74,12 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
       return callback(err);
     }
 
-    var unique = {};
-    var list = [];
-
-    for (let i = 0; i < options.txIdList.length; i++) {
-      unique[options.txIdList[i].txid + options.txIdList[i].height] = options.txIdList[i];
-    }
-
-    for (var prop in unique) {
-      list.push(unique[prop]);
-    }
-
-    options.txIdList = list.sort(function(a, b) {
-     return b.height - a.height;
+    var list = lodash.uniqBy(options.txIdList, function(x) { 
+      return x.txid + x.height;
     });
 
+
+    options.txIdList = lodash.sortBy(list,['height','txid']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -427,7 +417,6 @@ AddressService.prototype._getAddressTxHistory = function(options, callback) {
 };
 
 AddressService.prototype._getAddressTxidHistory = function(address, options, callback) {
-
   var self = this;
 
   options = options || {};
diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js
index 1fd2ddc19..895444056 100644
--- a/test/services/address/index.unit.js
+++ b/test/services/address/index.unit.js
@@ -8,6 +8,7 @@ var Encoding  = require('../../../lib/services/address/encoding');
 var Readable = require('stream').Readable;
 var EventEmitter = require('events').EventEmitter;
 var bcoin = require('bcoin');
+var lodash = require('lodash');
 
 describe('Address Service', function() {
 
@@ -54,7 +55,7 @@ describe('Address Service', function() {
 
   describe('#getAddressHistory', function() {
 
-    it('should get the address history', function(done) {
+    it('should get the address history (null case)', function(done) {
 
       sandbox.stub(addressService, '_getAddressTxidHistory').callsArgWith(2, null, null);
       sandbox.stub(addressService, '_getAddressTxHistory').callsArgWith(1, null, []);
@@ -74,6 +75,104 @@ describe('Address Service', function() {
       });
     });
 
+    it('should get the sorted address history', function(done) {
+
+      var old_getAddressTxidHistory = addressService._getAddressTxidHistory;
+      addressService._getAddressTxidHistory = function(addr, options, cb) {
+        options.txIdList = [
+          {
+            txid: "b",
+            height: 10,
+          },
+          {
+            txid: "a",
+            height: 10,
+          },
+          {
+            txid: "d",
+            height: 101,
+          },
+          {
+            txid: "c",
+            height: 100,
+          },
+        ];
+        return cb();
+      };
+
+
+      var old_getAddressTxHistory = addressService._getAddressTxHistory;
+      addressService._getAddressTxHistory = function(options, cb) {
+        return cb(null, options.txIdList);
+      };
+
+      addressService.getAddressHistory(['a', 'b', 'c'], { from: 12, to: 14 }, function(err, res) {
+
+        if (err) {
+          return done(err);
+        }
+
+        expect(res.totalCount).equal(4);
+        expect(lodash.map(res.items,'txid')).to.be.deep.equal(['a','b','c','d']);
+
+        addressService._getAddressTxidHistory = old_getAddressTxHistory;
+        addressService._getAddressTxHistory = old_getAddressTxHistory;
+        done();
+      });
+    });
+
+    it('should remove duplicated items in history', function(done) {
+
+      var old_getAddressTxidHistory = addressService._getAddressTxidHistory;
+      addressService._getAddressTxidHistory = function(addr, options, cb) {
+        options.txIdList = [
+          {
+            txid: "b",
+            height: 10,
+          },
+          {
+            txid: "b",
+            height: 10,
+          },
+          {
+            txid: "d",
+            height: 101,
+          },
+          {
+            txid: "c",
+            height: 100,
+          },
+         {
+            txid: "d",
+            height: 101,
+          },
+        ];
+        return cb();
+      };
+
+
+      var old_getAddressTxHistory = addressService._getAddressTxHistory;
+      addressService._getAddressTxHistory = function(options, cb) {
+        return cb(null, options.txIdList);
+      };
+
+      addressService.getAddressHistory(['a', 'b', 'c'], { from: 12, to: 14 }, function(err, res) {
+
+        if (err) {
+          return done(err);
+        }
+
+        expect(res.totalCount).equal(3);
+        expect(lodash.map(res.items,'txid')).to.be.deep.equal(['b','c','d']);
+
+        addressService._getAddressTxidHistory = old_getAddressTxHistory;
+        addressService._getAddressTxHistory = old_getAddressTxHistory;
+        done();
+      });
+    });
+
+
+
   });
 
   describe('#_getAddressTxidHistory', function() {

From 0c24271833885405b919e318c3e5ca3bca68f842 Mon Sep 17 00:00:00 2001
From: Matias Alejo Garcia <ematiu@gmail.com>
Date: Sun, 11 Feb 2018 19:47:52 -0300
Subject: [PATCH 06/15] change order to 'desc'

---
 lib/services/address/index.js       |  2 +-
 test/services/address/index.unit.js | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/services/address/index.js b/lib/services/address/index.js
index d48d82f13..97144473b 100644
--- a/lib/services/address/index.js
+++ b/lib/services/address/index.js
@@ -79,7 +79,7 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     });
 
 
-    options.txIdList = lodash.sortBy(list,['height','txid']);
+    options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js
index 895444056..5ded3800b 100644
--- a/test/services/address/index.unit.js
+++ b/test/services/address/index.unit.js
@@ -81,19 +81,19 @@ describe('Address Service', function() {
       addressService._getAddressTxidHistory = function(addr, options, cb) {
         options.txIdList = [
           {
-            txid: "b",
+            txid: "d",
             height: 10,
           },
           {
-            txid: "a",
+            txid: "c",
             height: 10,
           },
           {
-            txid: "d",
+            txid: "a",
             height: 101,
           },
           {
-            txid: "c",
+            txid: "b",
             height: 100,
           },
         ];
@@ -163,7 +163,7 @@ describe('Address Service', function() {
         }
 
         expect(res.totalCount).equal(3);
-        expect(lodash.map(res.items,'txid')).to.be.deep.equal(['b','c','d']);
+        expect(lodash.map(res.items,'txid')).to.be.deep.equal(['d','c','b']);
 
         addressService._getAddressTxidHistory = old_getAddressTxHistory;
         addressService._getAddressTxHistory = old_getAddressTxHistory;

From ddda913ccde8f4fc39f03accc84a649ac63e4f09 Mon Sep 17 00:00:00 2001
From: Matias Alejo Garcia <ematiu@gmail.com>
Date: Mon, 12 Feb 2018 00:48:30 -0300
Subject: [PATCH 07/15] add txIdList cache

---
 lib/services/address/index.js       |  95 +++++++++++++++++--
 package-lock.json                   | 141 ++++++++++++++++------------
 package.json                        |   5 +-
 test/services/address/index.unit.js | 118 +++++++++++++++++++++++
 4 files changed, 287 insertions(+), 72 deletions(-)

diff --git a/lib/services/address/index.js b/lib/services/address/index.js
index 97144473b..6f40da3bd 100644
--- a/lib/services/address/index.js
+++ b/lib/services/address/index.js
@@ -13,6 +13,17 @@ var Encoding = require('./encoding');
 var Transform = require('stream').Transform;
 var assert = require('assert');
 var utils = require('../../utils');
+var LRU = require('lru-cache');
+var XXHash = require('xxhash');
+
+
+
+// See rationale about this cache at function getTxList(next)
+const TXID_LIST_CACHE_ITEMS = 250;              // nr of items (this translates to: consecutive 
+                                                // clients downloading their tx history)
+const TXID_LIST_CACHE_EXPIRATION = 1000 * 30;   // ms
+const TXID_LIST_CACHE_MIN = 100;                // Min items to cache
+const TXID_LIST_CACHE_SEED = 0x3233DE;                // Min items to cache
 
 var AddressService = function(options) {
 
@@ -24,6 +35,11 @@ var AddressService = function(options) {
   this._network = this.node.network;
   this._db = this.node.services.db;
   this._mempool = this.node.services.mempool;
+  this._txIdListCache = new LRU({
+    max: TXID_LIST_CACHE_ITEMS,        
+    maxAge: TXID_LIST_CACHE_EXPIRATION 
+  });
+ 
 
   if (this._network === 'livenet') {
     this._network = 'main';
@@ -49,8 +65,12 @@ AddressService.dependencies = [
 // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
 // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
 // then I would pass back [tx1, tx2] in that order
+//
+// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
+//
 AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
   var self = this;
+  var cacheUsed = false;
 
   options = options || {};
   options.from = options.from || 0;
@@ -65,21 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     addresses = [addresses];
   }
 
-  async.eachLimit(addresses, 4, function(address, next) {
-    self._getAddressTxidHistory(address, options, next);
 
-  }, function(err) {
+  function getTxList(next) {
 
-    if(err) {
-      return callback(err);
+
+    function hashAddresses(addresses) {
+
+      // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
+      // a 32 bits hash is secure enough
+
+      return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
+    };
+
+    var calculatedCacheKey;
+
+    // We use the cache ONLY on from > 0 queries.
+    //
+    // Rationale: The a full history is downloaded, the client do
+    // from =0, to=x
+    // then from =x+1 to=y
+    // then [...]
+    // The objective of this cache is to speed up the from>0 queries, and also
+    // "freeze" the txid list during download.
+    //
+    if (options.from >0 ) {
+      
+      let cacheKey  = options.cacheKey;
+      if (!cacheKey) {
+         calculatedCacheKey = hashAddresses(addresses);
+         cacheKey =  calculatedCacheKey;
+      }
+
+      var txIdList  = self._txIdListCache.get(cacheKey);
+      if (txIdList) {
+        options.txIdList = txIdList;
+        cacheUsed = true;
+        return next();
+      }
     }
 
-    var list = lodash.uniqBy(options.txIdList, function(x) { 
-      return x.txid + x.height;
+    // Get the list from the db
+    async.eachLimit(addresses, 4, function(address, next) {
+      self._getAddressTxidHistory(address, options, next);
+    }, function(err) {
+      if (err) return next(err);
+
+      var list = lodash.uniqBy(options.txIdList, function(x) { 
+        return x.txid + x.height;
+      });
+
+
+      options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
+
+      if (list.length > TXID_LIST_CACHE_MIN) {
+        calculatedCacheKey  = calculatedCacheKey || hashAddresses(addresses);
+
+        self._txIdListCache.set(calculatedCacheKey, options.txIdList);
+      }
+
+      return next();
     });
 
+  };
+
+
+  getTxList(function(err) {
+    if(err) {
+      return callback(err);
+    }
 
-    options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -88,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
 
       var results = {
         totalCount: options.txIdList.length || 0,
-        items: txList
+        items: txList,
       };
 
-      callback(null, results);
+      // cacheUsed is returned for testing
+      callback(null, results, cacheUsed);
 
     });
   });
diff --git a/package-lock.json b/package-lock.json
index d8f28caa0..59692c09a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -159,6 +159,11 @@
         "lodash": "4.17.4"
       }
     },
+    "async-limiter": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
+      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -238,9 +243,9 @@
           "optional": true,
           "requires": {
             "debug": "2.6.8",
-            "engine.io": "3.1.0",
+            "engine.io": "3.1.4",
             "object-assign": "4.1.1",
-            "socket.io-adapter": "1.1.0",
+            "socket.io-adapter": "1.1.1",
             "socket.io-client": "2.0.1",
             "socket.io-parser": "3.1.2"
           }
@@ -256,7 +261,7 @@
             "component-bind": "1.0.0",
             "component-emitter": "1.2.1",
             "debug": "2.6.4",
-            "engine.io-client": "3.1.1",
+            "engine.io-client": "3.1.4",
             "has-cors": "1.1.0",
             "indexof": "0.0.1",
             "object-component": "0.0.3",
@@ -1351,50 +1356,84 @@
       }
     },
     "engine.io": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz",
-      "integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.4.tgz",
+      "integrity": "sha1-PQIRtwpVLOhB/8fahiezAamkFi4=",
       "optional": true,
       "requires": {
         "accepts": "1.3.3",
         "base64id": "1.0.0",
         "cookie": "0.3.1",
-        "debug": "2.6.8",
-        "engine.io-parser": "2.1.1",
+        "debug": "2.6.9",
+        "engine.io-parser": "2.1.2",
         "uws": "0.14.5",
-        "ws": "2.3.1"
+        "ws": "3.3.3"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
       }
     },
     "engine.io-client": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz",
-      "integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.4.tgz",
+      "integrity": "sha1-T88TcLRxY70s6b4nM5ckMDUNTqE=",
       "optional": true,
       "requires": {
         "component-emitter": "1.2.1",
         "component-inherit": "0.0.3",
-        "debug": "2.6.8",
-        "engine.io-parser": "2.1.1",
+        "debug": "2.6.9",
+        "engine.io-parser": "2.1.2",
         "has-cors": "1.1.0",
         "indexof": "0.0.1",
-        "parsejson": "0.0.3",
         "parseqs": "0.0.5",
         "parseuri": "0.0.5",
-        "ws": "2.3.1",
-        "xmlhttprequest-ssl": "1.5.3",
+        "ws": "3.3.3",
+        "xmlhttprequest-ssl": "1.5.5",
         "yeast": "0.1.2"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "xmlhttprequest-ssl": {
+          "version": "1.5.5",
+          "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
+          "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=",
+          "optional": true
+        }
       }
     },
     "engine.io-parser": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz",
-      "integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
+      "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
       "requires": {
         "after": "0.8.2",
-        "arraybuffer.slice": "0.0.6",
+        "arraybuffer.slice": "0.0.7",
         "base64-arraybuffer": "0.1.5",
         "blob": "0.0.4",
         "has-binary2": "1.0.2"
+      },
+      "dependencies": {
+        "arraybuffer.slice": {
+          "version": "0.0.7",
+          "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
+          "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
+        }
       }
     },
     "entities": {
@@ -4048,30 +4087,10 @@
       }
     },
     "socket.io-adapter": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.0.tgz",
-      "integrity": "sha1-x6pGUB3VVsLLiiivj/lcC14dqkw=",
-      "optional": true,
-      "requires": {
-        "debug": "2.3.3"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.3.3",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
-          "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
-          "optional": true,
-          "requires": {
-            "ms": "0.7.2"
-          }
-        },
-        "ms": {
-          "version": "0.7.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
-          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=",
-          "optional": true
-        }
-      }
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz",
+      "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=",
+      "optional": true
     },
     "socket.io-client": {
       "version": "1.7.4",
@@ -4483,9 +4502,9 @@
       "optional": true
     },
     "ultron": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz",
-      "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ="
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
+      "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
     },
     "unc-path-regex": {
       "version": "0.1.2",
@@ -4631,19 +4650,13 @@
       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
     },
     "ws": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz",
-      "integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
+      "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
       "requires": {
-        "safe-buffer": "5.0.1",
-        "ultron": "1.1.0"
-      },
-      "dependencies": {
-        "safe-buffer": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
-          "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c="
-        }
+        "async-limiter": "1.0.0",
+        "safe-buffer": "5.1.1",
+        "ultron": "1.1.1"
       }
     },
     "wtf-8": {
@@ -4661,6 +4674,14 @@
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
       "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
     },
+    "xxhash": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz",
+      "integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=",
+      "requires": {
+        "nan": "2.6.2"
+      }
+    },
     "yallist": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
diff --git a/package.json b/package.json
index 74dfa8037..a7e9c0698 100644
--- a/package.json
+++ b/package.json
@@ -43,12 +43,13 @@
     "levelup": "^2.0.0",
     "liftoff": "^2.2.0",
     "lodash": "^4.17.4",
-    "lru-cache": "^4.0.2",
+    "lru-cache": "^4.1.1",
     "memwatch-next": "^0.3.0",
     "mkdirp": "0.5.0",
     "path-is-absolute": "^1.0.0",
     "socket.io": "^1.4.5",
-    "socket.io-client": "^1.4.5"
+    "socket.io-client": "^1.4.5",
+    "xxhash": "^0.2.4"
   },
   "devDependencies": {
     "chai": "^3.5.0",
diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js
index 5ded3800b..afa7744b6 100644
--- a/test/services/address/index.unit.js
+++ b/test/services/address/index.unit.js
@@ -172,7 +172,125 @@ describe('Address Service', function() {
     });
 
 
+    describe('TxIdList cache', function() {
+      var list, old_getAddressTxidHistory, old_getAddressTxHistory;
+
+      beforeEach(function(done){
+        this.clock = sinon.useFakeTimers();
+        list = [];
+        for(let i=1000; i>0; i--) {
+          list.push({
+            txid: "txid" + i,
+            height: 1000  + i,
+
+          });
+        };
+        old_getAddressTxidHistory = addressService._getAddressTxidHistory;
+        // Note that this stub DOES NOT respect options.from/to as the real function
+        addressService._getAddressTxidHistory = function(addr, options, cb) {
+          options.txIdList = lodash.clone(list);
+          return cb();
+        };
+        old_getAddressTxHistory = addressService._getAddressTxHistory;
+
+        addressService._getAddressTxHistory = function(options, cb) {
+          return cb(null, options.txIdList);
+        };
+
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          expect(cacheUsed).equal(false);
+          done();
+        });
+      });
+
+      afterEach(function(done){
+        this.clock.restore();
+
+        addressService._getAddressTxidHistory = old_getAddressTxHistory;
+        addressService._getAddressTxHistory = old_getAddressTxHistory;
+        done();
+      });
+
 
+      it('should not cache the address txlist history when from =0 ', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          expect(cacheUsed).equal(false);
+          done();
+        });
+      });
+
+      it('should cache the address txlist history', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(true);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+
+      it('should retrive cached list using cachekey', function(done) {
+        addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(true);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+   
+
+      it('should expire cache', function(done) {
+        this.clock.tick(35*1000);
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(false);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+      it('should cache using the address as key', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(false);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+            if (err) {
+              return done(err);
+            }
+            expect(cacheUsed).equal(true);
+            expect(res.totalCount).equal(1000);
+            expect(res.items,'txid').to.be.deep.equal(list);
+            done();
+          });
+        });
+      });
+
+
+    });
   });
 
   describe('#_getAddressTxidHistory', function() {

From bf8f25d9eb35ad4d7588e64cf5a9bfa884830c0b Mon Sep 17 00:00:00 2001
From: Matias Alejo Garcia <ematiu@gmail.com>
Date: Thu, 12 Apr 2018 12:30:09 -0300
Subject: [PATCH 08/15] Revert "add txIdList cache"

---
 lib/services/address/index.js       |  95 ++-----------------
 package-lock.json                   | 141 ++++++++++++----------------
 package.json                        |   5 +-
 test/services/address/index.unit.js | 118 -----------------------
 4 files changed, 72 insertions(+), 287 deletions(-)

diff --git a/lib/services/address/index.js b/lib/services/address/index.js
index 6f40da3bd..97144473b 100644
--- a/lib/services/address/index.js
+++ b/lib/services/address/index.js
@@ -13,17 +13,6 @@ var Encoding = require('./encoding');
 var Transform = require('stream').Transform;
 var assert = require('assert');
 var utils = require('../../utils');
-var LRU = require('lru-cache');
-var XXHash = require('xxhash');
-
-
-
-// See rationale about this cache at function getTxList(next)
-const TXID_LIST_CACHE_ITEMS = 250;              // nr of items (this translates to: consecutive 
-                                                // clients downloading their tx history)
-const TXID_LIST_CACHE_EXPIRATION = 1000 * 30;   // ms
-const TXID_LIST_CACHE_MIN = 100;                // Min items to cache
-const TXID_LIST_CACHE_SEED = 0x3233DE;                // Min items to cache
 
 var AddressService = function(options) {
 
@@ -35,11 +24,6 @@ var AddressService = function(options) {
   this._network = this.node.network;
   this._db = this.node.services.db;
   this._mempool = this.node.services.mempool;
-  this._txIdListCache = new LRU({
-    max: TXID_LIST_CACHE_ITEMS,        
-    maxAge: TXID_LIST_CACHE_EXPIRATION 
-  });
- 
 
   if (this._network === 'livenet') {
     this._network = 'main';
@@ -65,12 +49,8 @@ AddressService.dependencies = [
 // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
 // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
 // then I would pass back [tx1, tx2] in that order
-//
-// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
-//
 AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
   var self = this;
-  var cacheUsed = false;
 
   options = options || {};
   options.from = options.from || 0;
@@ -85,75 +65,21 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     addresses = [addresses];
   }
 
+  async.eachLimit(addresses, 4, function(address, next) {
+    self._getAddressTxidHistory(address, options, next);
 
-  function getTxList(next) {
-
-
-    function hashAddresses(addresses) {
-
-      // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
-      // a 32 bits hash is secure enough
-
-      return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
-    };
+  }, function(err) {
 
-    var calculatedCacheKey;
-
-    // We use the cache ONLY on from > 0 queries.
-    //
-    // Rationale: The a full history is downloaded, the client do
-    // from =0, to=x
-    // then from =x+1 to=y
-    // then [...]
-    // The objective of this cache is to speed up the from>0 queries, and also
-    // "freeze" the txid list during download.
-    //
-    if (options.from >0 ) {
-      
-      let cacheKey  = options.cacheKey;
-      if (!cacheKey) {
-         calculatedCacheKey = hashAddresses(addresses);
-         cacheKey =  calculatedCacheKey;
-      }
-
-      var txIdList  = self._txIdListCache.get(cacheKey);
-      if (txIdList) {
-        options.txIdList = txIdList;
-        cacheUsed = true;
-        return next();
-      }
+    if(err) {
+      return callback(err);
     }
 
-    // Get the list from the db
-    async.eachLimit(addresses, 4, function(address, next) {
-      self._getAddressTxidHistory(address, options, next);
-    }, function(err) {
-      if (err) return next(err);
-
-      var list = lodash.uniqBy(options.txIdList, function(x) { 
-        return x.txid + x.height;
-      });
-
-
-      options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
-
-      if (list.length > TXID_LIST_CACHE_MIN) {
-        calculatedCacheKey  = calculatedCacheKey || hashAddresses(addresses);
-
-        self._txIdListCache.set(calculatedCacheKey, options.txIdList);
-      }
-
-      return next();
+    var list = lodash.uniqBy(options.txIdList, function(x) { 
+      return x.txid + x.height;
     });
 
-  };
-
-
-  getTxList(function(err) {
-    if(err) {
-      return callback(err);
-    }
 
+    options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -162,11 +88,10 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
 
       var results = {
         totalCount: options.txIdList.length || 0,
-        items: txList,
+        items: txList
       };
 
-      // cacheUsed is returned for testing
-      callback(null, results, cacheUsed);
+      callback(null, results);
 
     });
   });
diff --git a/package-lock.json b/package-lock.json
index 59692c09a..d8f28caa0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -159,11 +159,6 @@
         "lodash": "4.17.4"
       }
     },
-    "async-limiter": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
-      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
-    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -243,9 +238,9 @@
           "optional": true,
           "requires": {
             "debug": "2.6.8",
-            "engine.io": "3.1.4",
+            "engine.io": "3.1.0",
             "object-assign": "4.1.1",
-            "socket.io-adapter": "1.1.1",
+            "socket.io-adapter": "1.1.0",
             "socket.io-client": "2.0.1",
             "socket.io-parser": "3.1.2"
           }
@@ -261,7 +256,7 @@
             "component-bind": "1.0.0",
             "component-emitter": "1.2.1",
             "debug": "2.6.4",
-            "engine.io-client": "3.1.4",
+            "engine.io-client": "3.1.1",
             "has-cors": "1.1.0",
             "indexof": "0.0.1",
             "object-component": "0.0.3",
@@ -1356,84 +1351,50 @@
       }
     },
     "engine.io": {
-      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.4.tgz",
-      "integrity": "sha1-PQIRtwpVLOhB/8fahiezAamkFi4=",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz",
+      "integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=",
       "optional": true,
       "requires": {
         "accepts": "1.3.3",
         "base64id": "1.0.0",
         "cookie": "0.3.1",
-        "debug": "2.6.9",
-        "engine.io-parser": "2.1.2",
+        "debug": "2.6.8",
+        "engine.io-parser": "2.1.1",
         "uws": "0.14.5",
-        "ws": "3.3.3"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.6.9",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-          "optional": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        }
+        "ws": "2.3.1"
       }
     },
     "engine.io-client": {
-      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.4.tgz",
-      "integrity": "sha1-T88TcLRxY70s6b4nM5ckMDUNTqE=",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz",
+      "integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=",
       "optional": true,
       "requires": {
         "component-emitter": "1.2.1",
         "component-inherit": "0.0.3",
-        "debug": "2.6.9",
-        "engine.io-parser": "2.1.2",
+        "debug": "2.6.8",
+        "engine.io-parser": "2.1.1",
         "has-cors": "1.1.0",
         "indexof": "0.0.1",
+        "parsejson": "0.0.3",
         "parseqs": "0.0.5",
         "parseuri": "0.0.5",
-        "ws": "3.3.3",
-        "xmlhttprequest-ssl": "1.5.5",
+        "ws": "2.3.1",
+        "xmlhttprequest-ssl": "1.5.3",
         "yeast": "0.1.2"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.6.9",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-          "optional": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "xmlhttprequest-ssl": {
-          "version": "1.5.5",
-          "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
-          "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=",
-          "optional": true
-        }
       }
     },
     "engine.io-parser": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
-      "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz",
+      "integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=",
       "requires": {
         "after": "0.8.2",
-        "arraybuffer.slice": "0.0.7",
+        "arraybuffer.slice": "0.0.6",
         "base64-arraybuffer": "0.1.5",
         "blob": "0.0.4",
         "has-binary2": "1.0.2"
-      },
-      "dependencies": {
-        "arraybuffer.slice": {
-          "version": "0.0.7",
-          "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
-          "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
-        }
       }
     },
     "entities": {
@@ -4087,10 +4048,30 @@
       }
     },
     "socket.io-adapter": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz",
-      "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=",
-      "optional": true
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.0.tgz",
+      "integrity": "sha1-x6pGUB3VVsLLiiivj/lcC14dqkw=",
+      "optional": true,
+      "requires": {
+        "debug": "2.3.3"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.3.3",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
+          "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
+          "optional": true,
+          "requires": {
+            "ms": "0.7.2"
+          }
+        },
+        "ms": {
+          "version": "0.7.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
+          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=",
+          "optional": true
+        }
+      }
     },
     "socket.io-client": {
       "version": "1.7.4",
@@ -4502,9 +4483,9 @@
       "optional": true
     },
     "ultron": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
-      "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz",
+      "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ="
     },
     "unc-path-regex": {
       "version": "0.1.2",
@@ -4650,13 +4631,19 @@
       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
     },
     "ws": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
-      "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz",
+      "integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=",
       "requires": {
-        "async-limiter": "1.0.0",
-        "safe-buffer": "5.1.1",
-        "ultron": "1.1.1"
+        "safe-buffer": "5.0.1",
+        "ultron": "1.1.0"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
+          "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c="
+        }
       }
     },
     "wtf-8": {
@@ -4674,14 +4661,6 @@
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
       "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
     },
-    "xxhash": {
-      "version": "0.2.4",
-      "resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz",
-      "integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=",
-      "requires": {
-        "nan": "2.6.2"
-      }
-    },
     "yallist": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
diff --git a/package.json b/package.json
index a7e9c0698..74dfa8037 100644
--- a/package.json
+++ b/package.json
@@ -43,13 +43,12 @@
     "levelup": "^2.0.0",
     "liftoff": "^2.2.0",
     "lodash": "^4.17.4",
-    "lru-cache": "^4.1.1",
+    "lru-cache": "^4.0.2",
     "memwatch-next": "^0.3.0",
     "mkdirp": "0.5.0",
     "path-is-absolute": "^1.0.0",
     "socket.io": "^1.4.5",
-    "socket.io-client": "^1.4.5",
-    "xxhash": "^0.2.4"
+    "socket.io-client": "^1.4.5"
   },
   "devDependencies": {
     "chai": "^3.5.0",
diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js
index afa7744b6..5ded3800b 100644
--- a/test/services/address/index.unit.js
+++ b/test/services/address/index.unit.js
@@ -172,125 +172,7 @@ describe('Address Service', function() {
     });
 
 
-    describe('TxIdList cache', function() {
-      var list, old_getAddressTxidHistory, old_getAddressTxHistory;
-
-      beforeEach(function(done){
-        this.clock = sinon.useFakeTimers();
-        list = [];
-        for(let i=1000; i>0; i--) {
-          list.push({
-            txid: "txid" + i,
-            height: 1000  + i,
-
-          });
-        };
-        old_getAddressTxidHistory = addressService._getAddressTxidHistory;
-        // Note that this stub DOES NOT respect options.from/to as the real function
-        addressService._getAddressTxidHistory = function(addr, options, cb) {
-          options.txIdList = lodash.clone(list);
-          return cb();
-        };
-        old_getAddressTxHistory = addressService._getAddressTxHistory;
-
-        addressService._getAddressTxHistory = function(options, cb) {
-          return cb(null, options.txIdList);
-        };
-
-        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
-          if (err) {
-            return done(err);
-          }
-          expect(res.totalCount).equal(1000);
-          expect(res.items,'txid').to.be.deep.equal(list);
-          expect(cacheUsed).equal(false);
-          done();
-        });
-      });
-
-      afterEach(function(done){
-        this.clock.restore();
-
-        addressService._getAddressTxidHistory = old_getAddressTxHistory;
-        addressService._getAddressTxHistory = old_getAddressTxHistory;
-        done();
-      });
-
 
-      it('should not cache the address txlist history when from =0 ', function(done) {
-        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
-          if (err) {
-            return done(err);
-          }
-          expect(res.totalCount).equal(1000);
-          expect(res.items,'txid').to.be.deep.equal(list);
-          expect(cacheUsed).equal(false);
-          done();
-        });
-      });
-
-      it('should cache the address txlist history', function(done) {
-        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
-          if (err) {
-            return done(err);
-          }
-          expect(cacheUsed).equal(true);
-          expect(res.totalCount).equal(1000);
-          expect(res.items,'txid').to.be.deep.equal(list);
-          done();
-        });
-      });
-
-
-      it('should retrive cached list using cachekey', function(done) {
-        addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) {
-          if (err) {
-            return done(err);
-          }
-          expect(cacheUsed).equal(true);
-          expect(res.totalCount).equal(1000);
-          expect(res.items,'txid').to.be.deep.equal(list);
-          done();
-        });
-      });
-
-   
-
-      it('should expire cache', function(done) {
-        this.clock.tick(35*1000);
-        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
-          if (err) {
-            return done(err);
-          }
-          expect(cacheUsed).equal(false);
-          expect(res.totalCount).equal(1000);
-          expect(res.items,'txid').to.be.deep.equal(list);
-          done();
-        });
-      });
-
-      it('should cache using the address as key', function(done) {
-        addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
-          if (err) {
-            return done(err);
-          }
-          expect(cacheUsed).equal(false);
-          expect(res.totalCount).equal(1000);
-          expect(res.items,'txid').to.be.deep.equal(list);
-          addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
-            if (err) {
-              return done(err);
-            }
-            expect(cacheUsed).equal(true);
-            expect(res.totalCount).equal(1000);
-            expect(res.items,'txid').to.be.deep.equal(list);
-            done();
-          });
-        });
-      });
-
-
-    });
   });
 
   describe('#_getAddressTxidHistory', function() {

From 34f31ac57ddd2a420f3ed4bb0d3a7490d838741f Mon Sep 17 00:00:00 2001
From: Matias Alejo Garcia <ematiu@gmail.com>
Date: Thu, 12 Apr 2018 14:41:18 -0300
Subject: [PATCH 09/15] Revert "Revert "add txIdList cache""

---
 lib/services/address/index.js       |  95 +++++++++++++++++--
 package-lock.json                   | 141 ++++++++++++++++------------
 package.json                        |   5 +-
 test/services/address/index.unit.js | 118 +++++++++++++++++++++++
 4 files changed, 287 insertions(+), 72 deletions(-)

diff --git a/lib/services/address/index.js b/lib/services/address/index.js
index 97144473b..6f40da3bd 100644
--- a/lib/services/address/index.js
+++ b/lib/services/address/index.js
@@ -13,6 +13,17 @@ var Encoding = require('./encoding');
 var Transform = require('stream').Transform;
 var assert = require('assert');
 var utils = require('../../utils');
+var LRU = require('lru-cache');
+var XXHash = require('xxhash');
+
+
+
+// See rationale about this cache at function getTxList(next)
+const TXID_LIST_CACHE_ITEMS = 250;              // nr of items (this translates to: consecutive 
+                                                // clients downloading their tx history)
+const TXID_LIST_CACHE_EXPIRATION = 1000 * 30;   // ms
+const TXID_LIST_CACHE_MIN = 100;                // Min items to cache
+const TXID_LIST_CACHE_SEED = 0x3233DE;                // Min items to cache
 
 var AddressService = function(options) {
 
@@ -24,6 +35,11 @@ var AddressService = function(options) {
   this._network = this.node.network;
   this._db = this.node.services.db;
   this._mempool = this.node.services.mempool;
+  this._txIdListCache = new LRU({
+    max: TXID_LIST_CACHE_ITEMS,        
+    maxAge: TXID_LIST_CACHE_EXPIRATION 
+  });
+ 
 
   if (this._network === 'livenet') {
     this._network = 'main';
@@ -49,8 +65,12 @@ AddressService.dependencies = [
 // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
 // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
 // then I would pass back [tx1, tx2] in that order
+//
+// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
+//
 AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
   var self = this;
+  var cacheUsed = false;
 
   options = options || {};
   options.from = options.from || 0;
@@ -65,21 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     addresses = [addresses];
   }
 
-  async.eachLimit(addresses, 4, function(address, next) {
-    self._getAddressTxidHistory(address, options, next);
 
-  }, function(err) {
+  function getTxList(next) {
 
-    if(err) {
-      return callback(err);
+
+    function hashAddresses(addresses) {
+
+      // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
+      // a 32 bits hash is secure enough
+
+      return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
+    };
+
+    var calculatedCacheKey;
+
+    // We use the cache ONLY on from > 0 queries.
+    //
+    // Rationale: The a full history is downloaded, the client do
+    // from =0, to=x
+    // then from =x+1 to=y
+    // then [...]
+    // The objective of this cache is to speed up the from>0 queries, and also
+    // "freeze" the txid list during download.
+    //
+    if (options.from >0 ) {
+      
+      let cacheKey  = options.cacheKey;
+      if (!cacheKey) {
+         calculatedCacheKey = hashAddresses(addresses);
+         cacheKey =  calculatedCacheKey;
+      }
+
+      var txIdList  = self._txIdListCache.get(cacheKey);
+      if (txIdList) {
+        options.txIdList = txIdList;
+        cacheUsed = true;
+        return next();
+      }
     }
 
-    var list = lodash.uniqBy(options.txIdList, function(x) { 
-      return x.txid + x.height;
+    // Get the list from the db
+    async.eachLimit(addresses, 4, function(address, next) {
+      self._getAddressTxidHistory(address, options, next);
+    }, function(err) {
+      if (err) return next(err);
+
+      var list = lodash.uniqBy(options.txIdList, function(x) { 
+        return x.txid + x.height;
+      });
+
+
+      options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
+
+      if (list.length > TXID_LIST_CACHE_MIN) {
+        calculatedCacheKey  = calculatedCacheKey || hashAddresses(addresses);
+
+        self._txIdListCache.set(calculatedCacheKey, options.txIdList);
+      }
+
+      return next();
     });
 
+  };
+
+
+  getTxList(function(err) {
+    if(err) {
+      return callback(err);
+    }
 
-    options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -88,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
 
       var results = {
         totalCount: options.txIdList.length || 0,
-        items: txList
+        items: txList,
       };
 
-      callback(null, results);
+      // cacheUsed is returned for testing
+      callback(null, results, cacheUsed);
 
     });
   });
diff --git a/package-lock.json b/package-lock.json
index d8f28caa0..59692c09a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -159,6 +159,11 @@
         "lodash": "4.17.4"
       }
     },
+    "async-limiter": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
+      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -238,9 +243,9 @@
           "optional": true,
           "requires": {
             "debug": "2.6.8",
-            "engine.io": "3.1.0",
+            "engine.io": "3.1.4",
             "object-assign": "4.1.1",
-            "socket.io-adapter": "1.1.0",
+            "socket.io-adapter": "1.1.1",
             "socket.io-client": "2.0.1",
             "socket.io-parser": "3.1.2"
           }
@@ -256,7 +261,7 @@
             "component-bind": "1.0.0",
             "component-emitter": "1.2.1",
             "debug": "2.6.4",
-            "engine.io-client": "3.1.1",
+            "engine.io-client": "3.1.4",
             "has-cors": "1.1.0",
             "indexof": "0.0.1",
             "object-component": "0.0.3",
@@ -1351,50 +1356,84 @@
       }
     },
     "engine.io": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz",
-      "integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.4.tgz",
+      "integrity": "sha1-PQIRtwpVLOhB/8fahiezAamkFi4=",
       "optional": true,
       "requires": {
         "accepts": "1.3.3",
         "base64id": "1.0.0",
         "cookie": "0.3.1",
-        "debug": "2.6.8",
-        "engine.io-parser": "2.1.1",
+        "debug": "2.6.9",
+        "engine.io-parser": "2.1.2",
         "uws": "0.14.5",
-        "ws": "2.3.1"
+        "ws": "3.3.3"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
       }
     },
     "engine.io-client": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz",
-      "integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.4.tgz",
+      "integrity": "sha1-T88TcLRxY70s6b4nM5ckMDUNTqE=",
       "optional": true,
       "requires": {
         "component-emitter": "1.2.1",
         "component-inherit": "0.0.3",
-        "debug": "2.6.8",
-        "engine.io-parser": "2.1.1",
+        "debug": "2.6.9",
+        "engine.io-parser": "2.1.2",
         "has-cors": "1.1.0",
         "indexof": "0.0.1",
-        "parsejson": "0.0.3",
         "parseqs": "0.0.5",
         "parseuri": "0.0.5",
-        "ws": "2.3.1",
-        "xmlhttprequest-ssl": "1.5.3",
+        "ws": "3.3.3",
+        "xmlhttprequest-ssl": "1.5.5",
         "yeast": "0.1.2"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "xmlhttprequest-ssl": {
+          "version": "1.5.5",
+          "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
+          "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=",
+          "optional": true
+        }
       }
     },
     "engine.io-parser": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz",
-      "integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
+      "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
       "requires": {
         "after": "0.8.2",
-        "arraybuffer.slice": "0.0.6",
+        "arraybuffer.slice": "0.0.7",
         "base64-arraybuffer": "0.1.5",
         "blob": "0.0.4",
         "has-binary2": "1.0.2"
+      },
+      "dependencies": {
+        "arraybuffer.slice": {
+          "version": "0.0.7",
+          "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
+          "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
+        }
       }
     },
     "entities": {
@@ -4048,30 +4087,10 @@
       }
     },
     "socket.io-adapter": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.0.tgz",
-      "integrity": "sha1-x6pGUB3VVsLLiiivj/lcC14dqkw=",
-      "optional": true,
-      "requires": {
-        "debug": "2.3.3"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.3.3",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
-          "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
-          "optional": true,
-          "requires": {
-            "ms": "0.7.2"
-          }
-        },
-        "ms": {
-          "version": "0.7.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
-          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=",
-          "optional": true
-        }
-      }
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz",
+      "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=",
+      "optional": true
     },
     "socket.io-client": {
       "version": "1.7.4",
@@ -4483,9 +4502,9 @@
       "optional": true
     },
     "ultron": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz",
-      "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ="
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
+      "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
     },
     "unc-path-regex": {
       "version": "0.1.2",
@@ -4631,19 +4650,13 @@
       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
     },
     "ws": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz",
-      "integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
+      "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
       "requires": {
-        "safe-buffer": "5.0.1",
-        "ultron": "1.1.0"
-      },
-      "dependencies": {
-        "safe-buffer": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
-          "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c="
-        }
+        "async-limiter": "1.0.0",
+        "safe-buffer": "5.1.1",
+        "ultron": "1.1.1"
       }
     },
     "wtf-8": {
@@ -4661,6 +4674,14 @@
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
       "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
     },
+    "xxhash": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz",
+      "integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=",
+      "requires": {
+        "nan": "2.6.2"
+      }
+    },
     "yallist": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
diff --git a/package.json b/package.json
index 74dfa8037..a7e9c0698 100644
--- a/package.json
+++ b/package.json
@@ -43,12 +43,13 @@
     "levelup": "^2.0.0",
     "liftoff": "^2.2.0",
     "lodash": "^4.17.4",
-    "lru-cache": "^4.0.2",
+    "lru-cache": "^4.1.1",
     "memwatch-next": "^0.3.0",
     "mkdirp": "0.5.0",
     "path-is-absolute": "^1.0.0",
     "socket.io": "^1.4.5",
-    "socket.io-client": "^1.4.5"
+    "socket.io-client": "^1.4.5",
+    "xxhash": "^0.2.4"
   },
   "devDependencies": {
     "chai": "^3.5.0",
diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js
index 5ded3800b..afa7744b6 100644
--- a/test/services/address/index.unit.js
+++ b/test/services/address/index.unit.js
@@ -172,7 +172,125 @@ describe('Address Service', function() {
     });
 
 
+    describe('TxIdList cache', function() {
+      var list, old_getAddressTxidHistory, old_getAddressTxHistory;
+
+      beforeEach(function(done){
+        this.clock = sinon.useFakeTimers();
+        list = [];
+        for(let i=1000; i>0; i--) {
+          list.push({
+            txid: "txid" + i,
+            height: 1000  + i,
+
+          });
+        };
+        old_getAddressTxidHistory = addressService._getAddressTxidHistory;
+        // Note that this stub DOES NOT respect options.from/to as the real function
+        addressService._getAddressTxidHistory = function(addr, options, cb) {
+          options.txIdList = lodash.clone(list);
+          return cb();
+        };
+        old_getAddressTxHistory = addressService._getAddressTxHistory;
+
+        addressService._getAddressTxHistory = function(options, cb) {
+          return cb(null, options.txIdList);
+        };
+
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          expect(cacheUsed).equal(false);
+          done();
+        });
+      });
+
+      afterEach(function(done){
+        this.clock.restore();
+
+        addressService._getAddressTxidHistory = old_getAddressTxHistory;
+        addressService._getAddressTxHistory = old_getAddressTxHistory;
+        done();
+      });
+
 
+      it('should not cache the address txlist history when from =0 ', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          expect(cacheUsed).equal(false);
+          done();
+        });
+      });
+
+      it('should cache the address txlist history', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(true);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+
+      it('should retrive cached list using cachekey', function(done) {
+        addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(true);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+   
+
+      it('should expire cache', function(done) {
+        this.clock.tick(35*1000);
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(false);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+      it('should cache using the address as key', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(false);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+            if (err) {
+              return done(err);
+            }
+            expect(cacheUsed).equal(true);
+            expect(res.totalCount).equal(1000);
+            expect(res.items,'txid').to.be.deep.equal(list);
+            done();
+          });
+        });
+      });
+
+
+    });
   });
 
   describe('#_getAddressTxidHistory', function() {

From eb637d31252b7bcfc7047dd323b1dda403152fe5 Mon Sep 17 00:00:00 2001
From: Micah Riggan <mriggan@bitpay.com>
Date: Thu, 10 May 2018 16:32:08 -0400
Subject: [PATCH 10/15] Adding rpc for broadcastRawTransaction

---
 lib/services/p2p/index.js | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/lib/services/p2p/index.js b/lib/services/p2p/index.js
index 84ec2a745..a89d85b95 100644
--- a/lib/services/p2p/index.js
+++ b/lib/services/p2p/index.js
@@ -9,6 +9,7 @@ var BaseService = require('../../service');
 var assert = require('assert');
 var Bcoin = require('./bcoin');
 var BcoinTx = require('bcoin').tx;
+var BitcoreRPC = require('bitcoind-rpc');
 var Networks = require('bitcore-lib').Networks;
 var LRU = require('lru-cache');
 
@@ -21,6 +22,7 @@ var P2P = function(options) {
   BaseService.call(this, options);
   this._options = options;
 
+  this._initRPC(options);
   this._initP2P();
   this._initPubSub();
   this._bcoin = null;
@@ -128,26 +130,7 @@ P2P.prototype.getPublishEvents = function() {
 
 
 P2P.prototype.sendTransaction = function(tx, callback) {
-  var peer = this._getPeer();
-
-  var bcoinTx;
-  try {
-    bcoinTx = BcoinTx.fromRaw(tx, 'hex');
-  } catch(e) {
-    return callback(e);
-  }
-
-  log.info('P2P Service: sending transaction: ' + bcoinTx.txid());
-
-  this._outgoingTxs.set(bcoinTx.txid(), bcoinTx);
-  var inv = p2p.Inventory.forTransaction(bcoinTx.txid());
-  var txMessage = this.messages.Inventory([inv]);
-
-  peer.sendMessage(txMessage);
-
-  this._onPeerTx(peer, { transaction: bcoinTx });
-
-  return callback(null, bcoinTx.txid());
+  return this._client.sendRawTransaction(tx, callback);
 };
 
 
@@ -269,6 +252,17 @@ P2P.prototype._initCache = function() {
   this._inv = LRU(1000);
 };
 
+P2P.prototype._initRPC = function (options) {
+this._config = options.rpc || {
+    user: 'bitcoin',
+    pass: 'local321',
+    host: 'localhost',
+    protocol: 'http',
+    port: 8332
+  };
+  this._client = new BitcoreRPC(this._config);
+}
+
 P2P.prototype._initP2P = function() {
   this._maxPeers = this._options.maxPeers || 60;
   this._minPeers = this._options.minPeers || 0;

From 8fd99fe21c06ac5a20a45a87e06282e4a0c8545f Mon Sep 17 00:00:00 2001
From: Sky Young <sky@alexandria.io>
Date: Fri, 17 Aug 2018 12:48:39 -0700
Subject: [PATCH 11/15] Use live bcoinTx instead of running function on tx hex

---
 lib/services/p2p/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/services/p2p/index.js b/lib/services/p2p/index.js
index 928fdd588..de13c692c 100644
--- a/lib/services/p2p/index.js
+++ b/lib/services/p2p/index.js
@@ -141,7 +141,7 @@ P2P.prototype.sendTransaction = function(tx, callback) {
   log.info('P2P Service: sending transaction: ' + bcoinTx.txid());
 
   this._outgoingTxs.set(bcoinTx.txid(), bcoinTx);
-  var inv = p2p.Inventory.forTransaction(bcoinTx.txid(), tx.hasWitness());
+  var inv = p2p.Inventory.forTransaction(bcoinTx.txid(), bcoinTx.hasWitness());
   var txMessage = this.messages.Inventory([inv]);
 
   peer.sendMessage(txMessage);

From ee1e33628c430db8757d069fbc0da627e9058320 Mon Sep 17 00:00:00 2001
From: Sky Young <sky@alexandria.io>
Date: Fri, 17 Aug 2018 13:08:57 -0700
Subject: [PATCH 12/15] Use litecoind-rpc instead of bitcoind-rpc

---
 lib/services/p2p/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/services/p2p/index.js b/lib/services/p2p/index.js
index cd170e480..7874a4dfd 100644
--- a/lib/services/p2p/index.js
+++ b/lib/services/p2p/index.js
@@ -10,7 +10,7 @@ var assert = require('assert');
 var Bcoin = require('./bcoin'); // !!!: DO not change this to lcoin
 var BcoinTx = require('lcoin').tx;
 var Networks = require('litecore-lib').Networks;
-var BitcoreRPC = require('bitcoind-rpc');
+var BitcoreRPC = require('litecoind-rpc');
 var LRU = require('lru-cache');
 var constants = require('../../constants');
 

From 250f9e468535f438a7e7746f9ba1e94925ca30b2 Mon Sep 17 00:00:00 2001
From: Sky Young <sky@alexandria.io>
Date: Fri, 17 Aug 2018 13:35:05 -0700
Subject: [PATCH 13/15] Only add transaction to cache if it has at least 3
 confirmations

This prevents the transaction from getting stuck showing 0 confirmations as it currently does.
---
 lib/services/transaction/index.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/services/transaction/index.js b/lib/services/transaction/index.js
index 83bf40a57..99a790b4e 100644
--- a/lib/services/transaction/index.js
+++ b/lib/services/transaction/index.js
@@ -130,7 +130,8 @@ TransactionService.prototype.getTransaction = function(txid, options, callback)
     if (err) {
       return callback(err);
     }
-    if (tx) {
+    // Only add to cache if there are at least 3 confirmations, this prevents it getting stuck showing 0 confirmations
+    if (tx && tx.confirmations >= 3) {
       self._cacheTx.set(txid, tx);
     }
     callback(err, tx);

From 9048a08336cf2496c269ea58f40d6800180e4430 Mon Sep 17 00:00:00 2001
From: Sky Young <sky@alexandria.io>
Date: Fri, 17 Aug 2018 14:09:00 -0700
Subject: [PATCH 14/15] Use branch from `ostlerdev` instead of `ultragtx`

If the changes are merged into `ultragtx` then we should switch back, thus this temporary branch.
---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 92fed317a..936a32a82 100644
--- a/package.json
+++ b/package.json
@@ -29,7 +29,7 @@
   ],
   "dependencies": {
     "async": "^2.5.0",
-    "lcoin": "https://github.com/ultragtx/lcoin#for-litecore-v5.0",
+    "lcoin": "https://github.com/ostlerdev/lcoin#for-litecore-v5.0",
     "litecoind-rpc": "^0.7.1",
     "litecore-lib": "https://github.com/litecoin-project/litecore-lib#v5.0",
     "litecore-p2p": "https://github.com/litecoin-project/litecore-p2p#v5.0",

From f1f43892542b67a2cc14400dbbde602e29ac60bb Mon Sep 17 00:00:00 2001
From: Sky Young <sky@alexandria.io>
Date: Mon, 27 Aug 2018 13:35:47 -0700
Subject: [PATCH 15/15] Fix SyncComplete function

---
 lib/services/header/index.js | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/services/header/index.js b/lib/services/header/index.js
index fa736363b..d3a85267d 100644
--- a/lib/services/header/index.js
+++ b/lib/services/header/index.js
@@ -573,11 +573,15 @@ HeaderService.prototype._startBlockSubscription = function() {
 };
 
 HeaderService.prototype._syncComplete = function() {
+  // Check if we have reached the last block that we can download.
+  var bestHeight = Math.max(this._bestHeight, this._lastHeader.height);
+
+  var syncComplete = bestHeight === this._tip.height
 
-  // we always ask for the max number of headers, which is 2000.
-  // so any response with < 2000 means we have reached the end of the headers list.
-  // we could make an extra call if the number of total headers is multiple of 2000.
-  return this._lastHeaderCount < 2000;
+  if (syncComplete && this._initialSync)
+    log.info("Header Service: Sync Complete!")
+  
+  return syncComplete
 
 };