From 800ffb1fa31f063fad73ea970f6bb43bdfff6383 Mon Sep 17 00:00:00 2001 From: Dan Ports Date: Sun, 12 Apr 2020 17:30:31 -0700 Subject: [PATCH 1/6] #26: Fetch tree and blob contents lazily. --- apis/github | 80 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/apis/github b/apis/github index 5b6ab77..232b210 100644 --- a/apis/github +++ b/apis/github @@ -76,51 +76,84 @@ Blob.fullPath = function(self) return self.path end end +Blob.getContents = function(self) + local data = http.get( + ('https://raw.github.com/%s/%s/%s/%s'):format( + self.repo.user, self.repo.name, self.sha, + encodeURI(self:fullPath()) + ) + ) + return data.readAll() +end -- A class for a tree (aka a folder) local Tree = {} Tree.__index = Tree Tree.new = function(repo, sha, path) - local url = ('repos/%s/%s/git/trees/%s'):format(repo.user, repo.name, sha) - local status, data = getAPI(url, repo.auth) + return setmetatable({ + repo = repo, + sha = sha, + path = path or '' + }, Tree) +end +Tree.getContents = function(self) + if self.contents then + return self.contents + end + + local url = ('repos/%s/%s/git/trees/%s'):format(self.repo.user, self.repo.name, self.sha) + local status, data = getAPI(url, self.repo.auth) if not status then error('Could not get github API from ' ..url) end if data.tree then - local tree = setmetatable({ - repo=repo, sha=data.sha, - path=path or '', size=0, - contents={} - }, Tree) + self.sha = data.sha + self.size = 0 + self.contents = {} for _, childdata in ipairs(data.tree) do - childdata.fullPath = fs.combine(tree:fullPath(), childdata.path) + childdata.fullPath = fs.combine(self:fullPath(), childdata.path) local child if childdata.type == 'blob' then - child = Blob.new(repo, childdata.sha, childdata.path) + child = Blob.new(self.repo, childdata.sha, childdata.path) child.size = childdata.size elseif childdata.type == 'tree' then - child = Tree.new(repo, childdata.sha, childdata.path) + child = Tree.new(self.repo, childdata.sha, childdata.path) else - error("uh oh", JSON.encode(childdata)) - child = childdata + error("Unknown tree node type", JSON.encode(childdata)) end - tree.size = tree.size + child.size - child.parent = tree - table.insert(tree.contents, child) + self.size = self.size + (child.size or 0) + child.parent = self + table.insert(self.contents, child) end - return tree + return self.contents else - error("uh oh", JSON.encode(data)) + error("No tree data returned", JSON.encode(data)) end end local function walkTree(t, level) - for _, item in ipairs(t.contents) do + for _, item in ipairs(t:getContents()) do coroutine.yield(item, level) if getmetatable(item) == Tree then walkTree(item, level + 1) end end end +Tree.getFullSize = function(self) + local size = 0 + for item in self:iter() do + if getmetatable(item) == Blob then + size = size + item.size + end + end + return size +end +Tree.getChild = function(self, path) + for _, item in ipairs(self:getContents()) do + if item.path == path then + return item + end + end +end Tree.iter = function(self) return coroutine.wrap(function() walkTree(self, 0) @@ -134,19 +167,12 @@ Tree.cloneTo = function(self, dest, onProgress) end for item in self:iter() do - local gitpath = item:fullPath() - local path = fs.combine(dest, gitpath) + local path = fs.combine(dest, item:fullPath()) if getmetatable(item) == Tree then fs.makeDir(path) elseif getmetatable(item) == Blob then - local data = http.get( - ('https://raw.github.com/%s/%s/%s/%s'):format( - self.repo.user, self.repo.name, self.sha, - encodeURI(gitpath) - ) - ) + local text = item:getContents() local h = fs.open(path, 'w') - local text = data.readAll() h.write(text) h.close() end From f32b377fb6a2b5466796e5fd895d87b15029fbe0 Mon Sep 17 00:00:00 2001 From: Dan Ports Date: Sun, 19 Apr 2020 12:06:34 -0700 Subject: [PATCH 2/6] Fixed: Coroutine-based iterators do not play nice with the http.get API. --- apis/github | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/apis/github b/apis/github index 232b210..8b27470 100644 --- a/apis/github +++ b/apis/github @@ -130,14 +130,6 @@ Tree.getContents = function(self) error("No tree data returned", JSON.encode(data)) end end -local function walkTree(t, level) - for _, item in ipairs(t:getContents()) do - coroutine.yield(item, level) - if getmetatable(item) == Tree then - walkTree(item, level + 1) - end - end -end Tree.getFullSize = function(self) local size = 0 for item in self:iter() do @@ -155,9 +147,18 @@ Tree.getChild = function(self, path) end end Tree.iter = function(self) - return coroutine.wrap(function() - walkTree(self, 0) - end) + local stack = {{self, 0}} + return function() + local entry = table.remove(stack) + if entry then + if getmetatable(entry[1]) == Tree then + for _, child in ipairs(entry[1]:getContents()) do + table.insert(stack, {child, entry[2] + 1}) + end + end + return unpack(entry) + end + end end Tree.cloneTo = function(self, dest, onProgress) if not fs.exists(dest) then From 78baacfe499f04f8f134cf3bd15f023637961b7c Mon Sep 17 00:00:00 2001 From: Dan Ports Date: Sun, 19 Apr 2020 12:07:32 -0700 Subject: [PATCH 3/6] Fixed: raw.github.com needs the commit SHA rather than the blob SHA. --- apis/github | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/apis/github b/apis/github index 8b27470..6d61d01 100644 --- a/apis/github +++ b/apis/github @@ -89,11 +89,12 @@ end -- A class for a tree (aka a folder) local Tree = {} Tree.__index = Tree -Tree.new = function(repo, sha, path) +Tree.new = function(repo, sha, path, treeSha) return setmetatable({ repo = repo, sha = sha, - path = path or '' + path = path or '', + treeSha = treeSha }, Tree) end Tree.getContents = function(self) @@ -101,23 +102,27 @@ Tree.getContents = function(self) return self.contents end - local url = ('repos/%s/%s/git/trees/%s'):format(self.repo.user, self.repo.name, self.sha) + local url = ('repos/%s/%s/git/trees/%s'):format(self.repo.user, self.repo.name, self.treeSha or self.sha) local status, data = getAPI(url, self.repo.auth) if not status then - error('Could not get github API from ' ..url) + error('Could not get github API from ' .. url) end if data.tree then - self.sha = data.sha + -- Overwrite pseudo-SHAs like tag or branch names with actual SHAs. + if not self.treeSha then + self.sha = data.sha + self.treeSha = data.sha + end self.size = 0 self.contents = {} for _, childdata in ipairs(data.tree) do childdata.fullPath = fs.combine(self:fullPath(), childdata.path) local child if childdata.type == 'blob' then - child = Blob.new(self.repo, childdata.sha, childdata.path) + child = Blob.new(self.repo, self.sha, childdata.path) child.size = childdata.size elseif childdata.type == 'tree' then - child = Tree.new(self.repo, childdata.sha, childdata.path) + child = Tree.new(self.repo, self.sha, childdata.path, childdata.sha) else error("Unknown tree node type", JSON.encode(childdata)) end From bd4753cc6297a74fe02fef8f2c2a2d33d15e83b0 Mon Sep 17 00:00:00 2001 From: Dan Ports Date: Sun, 19 Apr 2020 12:08:55 -0700 Subject: [PATCH 4/6] Fixed: github client crashes because it does not use the new Tree:getFullSize() method. --- programs/github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/github b/programs/github index 676573c..b7428ee 100644 --- a/programs/github +++ b/programs/github @@ -147,7 +147,7 @@ if action == subcommands.clone then local tree = repo:tree(treeName) -- download the files - local totalSize = tree.size + local totalSize = tree:getFullSize() local freeSpace = fs.getFreeSpace(dest) if not hasEnoughSpace(totalSize, freeSpace) then From f1eda66ae97af26c9b5f97e1b5701674d2c7f84e Mon Sep 17 00:00:00 2001 From: Dan Ports Date: Sun, 19 Apr 2020 12:21:44 -0700 Subject: [PATCH 5/6] Make Tree.contents private, so to speak. --- apis/github | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apis/github b/apis/github index 6d61d01..ebb17d5 100644 --- a/apis/github +++ b/apis/github @@ -98,8 +98,8 @@ Tree.new = function(repo, sha, path, treeSha) }, Tree) end Tree.getContents = function(self) - if self.contents then - return self.contents + if self._contents then + return self._contents end local url = ('repos/%s/%s/git/trees/%s'):format(self.repo.user, self.repo.name, self.treeSha or self.sha) @@ -114,7 +114,7 @@ Tree.getContents = function(self) self.treeSha = data.sha end self.size = 0 - self.contents = {} + self._contents = {} for _, childdata in ipairs(data.tree) do childdata.fullPath = fs.combine(self:fullPath(), childdata.path) local child @@ -128,9 +128,9 @@ Tree.getContents = function(self) end self.size = self.size + (child.size or 0) child.parent = self - table.insert(self.contents, child) + table.insert(self._contents, child) end - return self.contents + return self._contents else error("No tree data returned", JSON.encode(data)) end From 01c9f37260ab653f9b06c68ac3ed8a762809c901 Mon Sep 17 00:00:00 2001 From: Dan Ports Date: Sun, 19 Apr 2020 18:23:38 -0700 Subject: [PATCH 6/6] #29: Check cache staleness whenever a tree is requested. --- apis/github | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/apis/github b/apis/github index ebb17d5..6a8b65e 100644 --- a/apis/github +++ b/apis/github @@ -198,20 +198,26 @@ Release.tree = function(self) end -- A class for a repository -local __repoPriv = setmetatable({}, {mode='k'}) local Repository = {} Repository.__index = Repository Repository.new = function(user, name, auth) - local r = setmetatable({user=user, name=name, auth=auth}, Repository) - __repoPriv[r] = {trees={}} - return r + return setmetatable({ + user = user, + name = name, + auth = auth, + _treeCache = {} + }, Repository) end Repository.tree = function(self, sha) sha = sha or "master" - if not __repoPriv[self].trees[sha] then - __repoPriv[self].trees[sha] = Tree.new(self, sha) + local cachedTree = self._treeCache[sha] + local newTree = Tree.new(self, sha) + newTree:getContents() -- Force resolution of commit-ish + if cachedTree and newTree.sha == cachedTree.sha then + return cachedTree end - return __repoPriv[self].trees[sha] + self._treeCache[sha] = newTree + return newTree end local function releaseFromURL(url, repo) local status, data = getAPI(url, repo.auth)