From 1a70f829455ab1af128eff0aa499ba5a7096943a Mon Sep 17 00:00:00 2001 From: Octal Date: Wed, 15 Jul 2020 14:21:40 -0600 Subject: [PATCH 01/11] Begin renaming variables, to make minified file readable --- automation/MakeZips.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index 2343435..f363f47 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -7,17 +7,17 @@ const exec = require('execa'); export const cwd = process.cwd(); export const folder = join(cwd, '..', '.divi'); -export const fin1 = join(folder, 'blocks'); -export const fin2 = join(folder, 'chainstate'); -export const fname1 = `blocks-snapshot.zip`; -export const fout1 = join(cwd, fname1); +export const blocksFolder = join(folder, 'blocks'); +export const chainstateFolder = join(folder, 'chainstate'); +export const blockZipFileName = `blocks-snapshot.zip`; +export const blockZipPath = join(cwd, blockZipFileName); -export const fname2 = `chainstate-snapshot.zip`; -export const fout2 = join(cwd, fname2); +export const chainstateZipFileName = `chainstate-snapshot.zip`; +export const chainstateZipPath = join(cwd, chainstateZipFileName); export const network = process.env.NETWORK || 'testnet'; -export const fname3 = Number(new Date()) + `-${network}-snapshot.zip`; -export const fout3 = join(cwd, fname3); +export const networkZipFileName = Number(new Date()) + `-${network}-snapshot.zip`; +export const networkZipPath = join(cwd, networkZipFileName); export const endpoint = 'https://nyc3.digitaloceanspaces.com'; export const accessKeyId = process.env.KEY || ''; @@ -34,28 +34,28 @@ const s3Stream = S3Stream(s3); export async function MakeZips() { console.log('Starting Zip process'); - const { stdout1, stderr1 } = await exec('zip', ['-1jr', fout1, fin1]); + const { stdout1, stderr1 } = await exec('zip', ['-1jr', blockZipPath, blocksFolder]); console.log(stdout1, stderr1); console.log('Zipped blocks'); - const { stdout2, stderr2 } = await exec('zip', ['-1jr', fout2, fin2]); + const { stdout2, stderr2 } = await exec('zip', ['-1jr', chainstateZipPath, chainstateFolder]); console.log(stdout2, stderr2); console.log('Zipped chainstate'); - const { stdout3, stderr3 } = await exec('zip', ['-1jr', fout3, fout1, fout2]); + const { stdout3, stderr3 } = await exec('zip', ['-1jr', networkZipPath, blockZipPath, chainstateZipPath]); console.log(stdout3, stderr3); console.log('Zipped all files'); - const reader = fs.createReadStream(fout3); + const reader = fs.createReadStream(networkZipPath); const uploader = s3Stream.upload( { ACL: 'public-read', ContentType: 'application/zip', Bucket: 'divi-snapshots', - Key: fname3, + Key: networkZipFileName, }, ); @@ -71,9 +71,9 @@ export async function MakeZips() { console.log(data); console.log('Snapshot taken successfully'); - fs.unlinkSync(fout1); - fs.unlinkSync(fout2); - fs.unlinkSync(fout3); + fs.unlinkSync(blockZipPath); + fs.unlinkSync(chainstateZipPath); + fs.unlinkSync(networkZipPath); console.log('Removed temporary zip files'); }); From cac5dd3f4a7b4e4f4c944dffbe8bb68d356e5290 Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 10:18:54 -0600 Subject: [PATCH 02/11] Logs --- automation/MakeZips.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index f363f47..e8d5b4e 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -34,19 +34,19 @@ const s3Stream = S3Stream(s3); export async function MakeZips() { console.log('Starting Zip process'); + console.log("Starting Block zip process"); const { stdout1, stderr1 } = await exec('zip', ['-1jr', blockZipPath, blocksFolder]); console.log(stdout1, stderr1); - console.log('Zipped blocks'); + console.log("Starting Chainstate zip process"); const { stdout2, stderr2 } = await exec('zip', ['-1jr', chainstateZipPath, chainstateFolder]); console.log(stdout2, stderr2); - console.log('Zipped chainstate'); + console.log("Starting zip for all files"); const { stdout3, stderr3 } = await exec('zip', ['-1jr', networkZipPath, blockZipPath, chainstateZipPath]); console.log(stdout3, stderr3); - console.log('Zipped all files'); const reader = fs.createReadStream(networkZipPath); From ae3ae6ba4b26b0d1d0fa273a9a14f927379b2ef6 Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 10:20:51 -0600 Subject: [PATCH 03/11] Log keys that actually exist --- automation/MakeZips.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index e8d5b4e..f0e0ab6 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -35,18 +35,18 @@ export async function MakeZips() { console.log('Starting Zip process'); console.log("Starting Block zip process"); - const { stdout1, stderr1 } = await exec('zip', ['-1jr', blockZipPath, blocksFolder]); - console.log(stdout1, stderr1); + var { stdout, stderr } = await exec('zip', ['-1jr', blockZipPath, blocksFolder]); + console.log(stdout, stderr); console.log('Zipped blocks'); console.log("Starting Chainstate zip process"); - const { stdout2, stderr2 } = await exec('zip', ['-1jr', chainstateZipPath, chainstateFolder]); - console.log(stdout2, stderr2); + var { stdout, stderr } = await exec('zip', ['-1jr', chainstateZipPath, chainstateFolder]); + console.log(stdout, stderr); console.log('Zipped chainstate'); console.log("Starting zip for all files"); - const { stdout3, stderr3 } = await exec('zip', ['-1jr', networkZipPath, blockZipPath, chainstateZipPath]); - console.log(stdout3, stderr3); + var { stdout, stderr } = await exec('zip', ['-1jr', networkZipPath, blockZipPath, chainstateZipPath]); + console.log(stdout, stderr); console.log('Zipped all files'); const reader = fs.createReadStream(networkZipPath); From 6d532bb3e28c0a41b256d41110834c96cd18c293 Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 15:28:07 -0600 Subject: [PATCH 04/11] Organize --- automation/MakeZips.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index f0e0ab6..a7c3cb2 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -7,11 +7,12 @@ const exec = require('execa'); export const cwd = process.cwd(); export const folder = join(cwd, '..', '.divi'); + export const blocksFolder = join(folder, 'blocks'); -export const chainstateFolder = join(folder, 'chainstate'); export const blockZipFileName = `blocks-snapshot.zip`; export const blockZipPath = join(cwd, blockZipFileName); +export const chainstateFolder = join(folder, 'chainstate'); export const chainstateZipFileName = `chainstate-snapshot.zip`; export const chainstateZipPath = join(cwd, chainstateZipFileName); From c89403dacba0b824b77121140c28c1a29d61a776 Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 15:31:05 -0600 Subject: [PATCH 05/11] Rename variable --- automation/MakeZips.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index a7c3cb2..3ebcfbe 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -17,8 +17,8 @@ export const chainstateZipFileName = `chainstate-snapshot.zip`; export const chainstateZipPath = join(cwd, chainstateZipFileName); export const network = process.env.NETWORK || 'testnet'; -export const networkZipFileName = Number(new Date()) + `-${network}-snapshot.zip`; -export const networkZipPath = join(cwd, networkZipFileName); +export const snapshotZipFileName = Number(new Date()) + `-${network}-snapshot.zip`; +export const networkZipPath = join(cwd, snapshotZipFileName); export const endpoint = 'https://nyc3.digitaloceanspaces.com'; export const accessKeyId = process.env.KEY || ''; @@ -56,7 +56,7 @@ export async function MakeZips() { ACL: 'public-read', ContentType: 'application/zip', Bucket: 'divi-snapshots', - Key: networkZipFileName, + Key: snapshotZipFileName, }, ); From 359aaee492db889ca969e6e9c64584560fe107db Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 15:31:25 -0600 Subject: [PATCH 06/11] Rename Variable --- automation/MakeZips.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index 3ebcfbe..fcf29a3 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -18,7 +18,7 @@ export const chainstateZipPath = join(cwd, chainstateZipFileName); export const network = process.env.NETWORK || 'testnet'; export const snapshotZipFileName = Number(new Date()) + `-${network}-snapshot.zip`; -export const networkZipPath = join(cwd, snapshotZipFileName); +export const snapshotZipFilePath = join(cwd, snapshotZipFileName); export const endpoint = 'https://nyc3.digitaloceanspaces.com'; export const accessKeyId = process.env.KEY || ''; @@ -46,11 +46,11 @@ export async function MakeZips() { console.log('Zipped chainstate'); console.log("Starting zip for all files"); - var { stdout, stderr } = await exec('zip', ['-1jr', networkZipPath, blockZipPath, chainstateZipPath]); + var { stdout, stderr } = await exec('zip', ['-1jr', snapshotZipFilePath, blockZipPath, chainstateZipPath]); console.log(stdout, stderr); console.log('Zipped all files'); - const reader = fs.createReadStream(networkZipPath); + const reader = fs.createReadStream(snapshotZipFilePath); const uploader = s3Stream.upload( { ACL: 'public-read', @@ -74,7 +74,7 @@ export async function MakeZips() { fs.unlinkSync(blockZipPath); fs.unlinkSync(chainstateZipPath); - fs.unlinkSync(networkZipPath); + fs.unlinkSync(snapshotZipFilePath); console.log('Removed temporary zip files'); }); From ccdffc8b6593cd7fd68888fa818ce71dff1e9540 Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 15:31:45 -0600 Subject: [PATCH 07/11] temporary folder name --- automation/MakeZips.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index fcf29a3..e80405c 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -20,6 +20,8 @@ export const network = process.env.NETWORK || 'testnet'; export const snapshotZipFileName = Number(new Date()) + `-${network}-snapshot.zip`; export const snapshotZipFilePath = join(cwd, snapshotZipFileName); +export const temporaryDiviSnapshotFolderPath = "/divi-snapshot"; + export const endpoint = 'https://nyc3.digitaloceanspaces.com'; export const accessKeyId = process.env.KEY || ''; export const secretAccessKey = process.env.SECRET || ''; From 4ad84fae57b896d41098f809fcdd0299b083f37c Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 15:32:52 -0600 Subject: [PATCH 08/11] create snapshot folder --- automation/MakeZips.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index e80405c..5ef41ec 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -37,6 +37,11 @@ const s3Stream = S3Stream(s3); export async function MakeZips() { console.log('Starting Zip process'); + console.log("creating snapshot folder", temporaryDiviSnapshotFolderPath); + var { stdout, stderr } = await exec('mkdir', [temporaryDiviSnapshotFolderPath]); + console.log(stdout, stderr); + console.log("created snapshot folder"); + console.log("Starting Block zip process"); var { stdout, stderr } = await exec('zip', ['-1jr', blockZipPath, blocksFolder]); console.log(stdout, stderr); From 0afaf72a4517006b2571df10b2d5a9e9b8e0e84c Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 15:36:11 -0600 Subject: [PATCH 09/11] Copy blocks & chainstate into folder, then zip. --- automation/MakeZips.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index 5ef41ec..681e907 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -42,21 +42,29 @@ export async function MakeZips() { console.log(stdout, stderr); console.log("created snapshot folder"); - console.log("Starting Block zip process"); - var { stdout, stderr } = await exec('zip', ['-1jr', blockZipPath, blocksFolder]); + console.log("copying blocks into snapshot folder"); + var { stdout, stderr } = await exec('cp', ['-a', blocksFolder, temporaryDiviSnapshotFolderPath+"/blocks"]); console.log(stdout, stderr); console.log('Zipped blocks'); - console.log("Starting Chainstate zip process"); - var { stdout, stderr } = await exec('zip', ['-1jr', chainstateZipPath, chainstateFolder]); + console.log("copying chainstate into snapshot folder"); + var { stdout, stderr } = await exec('cp', ['-a', chainstateFolder, temporaryDiviSnapshotFolderPath+"/chainstate"]); console.log(stdout, stderr); console.log('Zipped chainstate'); - console.log("Starting zip for all files"); - var { stdout, stderr } = await exec('zip', ['-1jr', snapshotZipFilePath, blockZipPath, chainstateZipPath]); + console.log("Zipping snapshot folder"); + var { stdout, stderr } = await exec('zip', ['-1r', snapshotZipFileName, temporaryDiviSnapshotFolderPath]); console.log(stdout, stderr); - console.log('Zipped all files'); + console.log('Zipped'); + console.log("Deleting temporary snapshot folder"); + var { stdout, stderr } = await exec('rm', ['-r', temporaryDiviSnapshotFolderPath]); + console.log(stdout, stderr); + console.log('Deleted'); + + console.log('Zip process completed'); + + console.log('Uploading'); const reader = fs.createReadStream(snapshotZipFilePath); const uploader = s3Stream.upload( { @@ -79,8 +87,6 @@ export async function MakeZips() { console.log(data); console.log('Snapshot taken successfully'); - fs.unlinkSync(blockZipPath); - fs.unlinkSync(chainstateZipPath); fs.unlinkSync(snapshotZipFilePath); console.log('Removed temporary zip files'); From ab5f803b0cd1e6b7f87c92fa41caf9ecdb48a5d9 Mon Sep 17 00:00:00 2001 From: Octal Date: Fri, 17 Jul 2020 16:03:59 -0600 Subject: [PATCH 10/11] rename file --- automation/MakeZips.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index 681e907..52e77c4 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -17,7 +17,7 @@ export const chainstateZipFileName = `chainstate-snapshot.zip`; export const chainstateZipPath = join(cwd, chainstateZipFileName); export const network = process.env.NETWORK || 'testnet'; -export const snapshotZipFileName = Number(new Date()) + `-${network}-snapshot.zip`; +export const snapshotZipFileName = `divi-snapshots-${Number(new Date())}.zip`; export const snapshotZipFilePath = join(cwd, snapshotZipFileName); export const temporaryDiviSnapshotFolderPath = "/divi-snapshot"; From c8e7a8230d36d0f27a5e29a02172816320824165 Mon Sep 17 00:00:00 2001 From: Octal Date: Mon, 20 Jul 2020 15:40:37 -0600 Subject: [PATCH 11/11] Fix name --- automation/MakeZips.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/MakeZips.ts b/automation/MakeZips.ts index 52e77c4..6390fe6 100644 --- a/automation/MakeZips.ts +++ b/automation/MakeZips.ts @@ -17,7 +17,7 @@ export const chainstateZipFileName = `chainstate-snapshot.zip`; export const chainstateZipPath = join(cwd, chainstateZipFileName); export const network = process.env.NETWORK || 'testnet'; -export const snapshotZipFileName = `divi-snapshots-${Number(new Date())}.zip`; +export const snapshotZipFileName = `${Number(new Date())}-${network}-snapshot.zip`; export const snapshotZipFilePath = join(cwd, snapshotZipFileName); export const temporaryDiviSnapshotFolderPath = "/divi-snapshot";