From 0ca3e240cc0d3c57726df733e60a479258d6c835 Mon Sep 17 00:00:00 2001 From: Roman Meszaros Date: Tue, 18 Oct 2022 13:41:49 +0200 Subject: [PATCH 1/3] FIX TEST Multiple files cross-contamination extraction --- src/extractor/fileExtractor.ts | 18 ++++++++++++++++++ test/services/zipToSQLite.test.ts | 24 ++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/extractor/fileExtractor.ts b/src/extractor/fileExtractor.ts index 9556a82..a389a29 100644 --- a/src/extractor/fileExtractor.ts +++ b/src/extractor/fileExtractor.ts @@ -90,4 +90,22 @@ export class FileExtractor { database.createTable(this.serviceName, this.table); } } + + /** + * Used mainly for testing. When multiple files for the same service are + * processed needs to dump the extractors data between different files. + * @param serviceName + */ + static dumpExtractorData(serviceName: string) { + if (!FileExtractor.registeredExtractors[serviceName]) { + throw new Error(`Extractors for service ${serviceName} do not exist`); + } + + // eslint-disable-next-line no-restricted-syntax + for (const extractorEntry of FileExtractor.registeredExtractors[ + serviceName + ]) { + extractorEntry.extractor.table.rows = []; + } + } } diff --git a/test/services/zipToSQLite.test.ts b/test/services/zipToSQLite.test.ts index 09a021d..7fdbcda 100644 --- a/test/services/zipToSQLite.test.ts +++ b/test/services/zipToSQLite.test.ts @@ -1,5 +1,6 @@ import * as zip from "@zip.js/zip.js"; +import { FileExtractor } from "../../src/extractor/fileExtractor"; import { zipToSQLiteInstance } from "../../src/services/zipToSQLite"; import { deleteFile, @@ -56,14 +57,21 @@ describe("services/zipExporter", () => { expect(fileExists(`zip/${serviceName}/${testFile}.sqlite`)).toBeTruthy(); }; - const testPromises: Promise[] = []; - Object.entries(testFiles).forEach((service: [string, string[]]) => { - const serviceName = service[0]; - const testFilesForService = service[1]; - testFilesForService.forEach((testFile: string) => { - testPromises.push(runTestAgainstFile(serviceName, testFile)); - }); - }); + const testPromises = Object.entries(testFiles).map( + async (service: [string, string[]]) => { + const serviceName = service[0]; + const testFilesForService = service[1]; + + // Extracting data from different zip files of the same service type + // needs to be done in sequence. Extractor table data needs to be + // cleared before another zip file's data get extracted. + // eslint-disable-next-line no-restricted-syntax + for (const testFile of testFilesForService) { + FileExtractor.dumpExtractorData(serviceName); + await runTestAgainstFile(serviceName, testFile); + } + } + ); await Promise.all(testPromises); }); From 2a9b7bd3b8c3231a519fc4261b17e20b01a4e49a Mon Sep 17 00:00:00 2001 From: Roman Meszaros Date: Wed, 19 Oct 2022 13:14:28 +0200 Subject: [PATCH 2/3] REFACTOR AmazonOrdersCsv - no orders --- src/extractor/amazonOrders/csv/ordersCsv.ts | 102 +++++++++--------- src/extractor/csvExtractor.ts | 6 +- .../amazonOrders/csv/ordersCsv.test.ts | 87 +++++++++++++-- 3 files changed, 132 insertions(+), 63 deletions(-) diff --git a/src/extractor/amazonOrders/csv/ordersCsv.ts b/src/extractor/amazonOrders/csv/ordersCsv.ts index 4951cce..dde5104 100644 --- a/src/extractor/amazonOrders/csv/ordersCsv.ts +++ b/src/extractor/amazonOrders/csv/ordersCsv.ts @@ -7,59 +7,59 @@ const extractPrice = (price: string) => class OrdersCsv extends CsvExtractor { async process() { - if (this.fileContents.includes("No data found for this time period")) { - return; - } - - await this.parse(); + try { + await this.parse(); - if (this.csvDocument) { - const mappedRows = this.csvDocument.map( - (order: any) => - new Orders({ - orderDate: order["order date"], - orderId: order["order id"], - title: order.title, - category: order.category, - asinIsbn: order["asin/isbn"], - unspscCode: order["unspsc code"], - website: order.website, - releaseDate: order["release date"], - condition: order.condition, - seller: order.seller, - sellerCredentials: order["seller credentials"], - listPricePerUnit: extractPrice(order["list price per unit"]), - purchasePricePerUnit: extractPrice( - order["purchase price per unit"] - ), - quantity: order.quantity, - paymentInstrumentType: order["payment instrument type"], - purchaseOrderNumber: order["purchase order number"], - poLineNumber: order["po line number"], - orderingCustomerEmail: order["ordering customer email"], - shipmentDate: order["shipment date"], - shippingAddressName: order["shipping address name"], - shippingAddressStreet1: order["shipping address street 1"], - shippingAddressStreet2: order["shipping address street 2"], - shippingAddressCity: order["shipping address city"], - shippingAddressState: order["shipping address state"], - shippingAddressZip: order["shipping address zip"], - orderStatus: order["order status"], - carrierNameAndTrackingNumber: - order["carrier name & tracking number"], - itemSubtotal: extractPrice(order["item subtotal"]), - itemSubtotalTax: extractPrice(order["item subtotal tax"]), - itemTotal: extractPrice(order["item total"]), - taxExemptionApplied: order["tax exemption applied"], - taxExemptionType: order["tax exemption type"], - exemptionOptOut: order["exemption opt-out"], - buyerName: order["buyer name"], - currency: order.currency, - groupName: order["group name"], - }) - ); + if (this.csvDocument) { + const mappedRows = this.csvDocument.map( + (order: any) => + new Orders({ + orderDate: order["order date"], + orderId: order["order id"], + title: order.title, + category: order.category, + asinIsbn: order["asin/isbn"], + unspscCode: order["unspsc code"], + website: order.website, + releaseDate: order["release date"], + condition: order.condition, + seller: order.seller, + sellerCredentials: order["seller credentials"], + listPricePerUnit: extractPrice(order["list price per unit"]), + purchasePricePerUnit: extractPrice( + order["purchase price per unit"] + ), + quantity: order.quantity, + paymentInstrumentType: order["payment instrument type"], + purchaseOrderNumber: order["purchase order number"], + poLineNumber: order["po line number"], + orderingCustomerEmail: order["ordering customer email"], + shipmentDate: order["shipment date"], + shippingAddressName: order["shipping address name"], + shippingAddressStreet1: order["shipping address street 1"], + shippingAddressStreet2: order["shipping address street 2"], + shippingAddressCity: order["shipping address city"], + shippingAddressState: order["shipping address state"], + shippingAddressZip: order["shipping address zip"], + orderStatus: order["order status"], + carrierNameAndTrackingNumber: + order["carrier name & tracking number"], + itemSubtotal: extractPrice(order["item subtotal"]), + itemSubtotalTax: extractPrice(order["item subtotal tax"]), + itemTotal: extractPrice(order["item total"]), + taxExemptionApplied: order["tax exemption applied"], + taxExemptionType: order["tax exemption type"], + exemptionOptOut: order["exemption opt-out"], + buyerName: order["buyer name"], + currency: order.currency, + groupName: order["group name"], + }) + ); - this.table.rows.push(...mappedRows); + this.table.rows.push(...mappedRows); + } + } catch (e) { + console.warn(e); } } } diff --git a/src/extractor/csvExtractor.ts b/src/extractor/csvExtractor.ts index aa63f49..a9935d2 100644 --- a/src/extractor/csvExtractor.ts +++ b/src/extractor/csvExtractor.ts @@ -15,7 +15,7 @@ export class CsvExtractor extends FileExtractor { text: string, options?: Partial ) { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const csvConfig: csvParser.ParseWorkerConfig = { // Default CSV parsing options header: true, @@ -24,11 +24,11 @@ export class CsvExtractor extends FileExtractor { complete: (results: csvParser.ParseResult) => { if (results.errors?.length > 0) { logger.warn( - `Error parsing ${this.zipEntry.data.filename}`, + `Error parsing ${this.zipEntry?.data?.filename}`, results.errors ); + reject(results.errors); } - resolve(caseInsensitiveWrapper(results.data) as []); }, diff --git a/test/extractor/amazonOrders/csv/ordersCsv.test.ts b/test/extractor/amazonOrders/csv/ordersCsv.test.ts index 6a80bb6..fdd83b3 100644 --- a/test/extractor/amazonOrders/csv/ordersCsv.test.ts +++ b/test/extractor/amazonOrders/csv/ordersCsv.test.ts @@ -1,7 +1,21 @@ +import config from "../../../../src/config"; import { ordersCsv } from "../../../../src/extractor/amazonOrders/csv/ordersCsv"; +import { FileExtractor } from "../../../../src/extractor/fileExtractor"; +import { + BoolTableValue, + DateTableValue, + EmailTableValue, + FloatTableValue, + IntegerTableValue, + TextTableValue, + UrlTableValue, +} from "../../../../src/models/table"; import { loadTestFileAsText } from "../../../helper"; describe("Orders (CSV)", () => { + beforeEach(() => { + FileExtractor.dumpExtractorData(config.SERVICE_AMAZON_ORDERS); + }); test("it should load file correctly", async () => { const data = await loadTestFileAsText( "/csv/amazon_orders/Amazon_Orders_csv.csv" @@ -13,14 +27,69 @@ describe("Orders (CSV)", () => { const { rows } = ordersCsv.table; expect(rows.length).toBe(2); - // expect(rows).toMatchSnapshot( - // [ - // { - // order_date: expect.any(DateTableValue), - // shipment_date: expect.any(DateTableValue), - // }, - // ], - // "ordersCsv.test.ts.snap" - // ); + + expect(rows[0].asin_isbn).toEqual(new TextTableValue("B09JR24YYW")); + expect(rows[0].buyer_name).toEqual(new TextTableValue("Z Hay")); + expect(rows[0].carrier_name_and_tracking_number).toEqual( + new TextTableValue("AMZN_US(TBA154327887604)") + ); + expect(rows[0].category).toEqual(new TextTableValue("CHARGING_ADAPTER")); + expect(rows[0].condition).toEqual(new TextTableValue("new")); + expect(rows[0].currency).toEqual(new TextTableValue("USD")); + expect(rows[0].exemption_opt_out).toEqual(new BoolTableValue(false)); + expect(rows[0].group_name).toEqual(new TextTableValue("")); + expect(rows[0].item_subtotal).toEqual(new FloatTableValue(59)); + expect(rows[0].item_subtotal_tax).toEqual(new FloatTableValue(5.22)); + expect(rows[0].item_total).toEqual(new FloatTableValue(64.22)); + expect(rows[0].list_price_per_unit).toEqual(new FloatTableValue(59)); + expect(rows[0].order_date).toEqual(new DateTableValue("01/11/22")); + expect(rows[0].order_id).toEqual(new TextTableValue("113-9614127-1313059")); + expect(rows[0].order_status).toEqual(new TextTableValue("Shipped")); + expect(rows[0].ordering_customer_email).toEqual( + new EmailTableValue("zach@gmail.com") + ); + expect(rows[0].payment_instrument_type).toEqual( + new TextTableValue("Visa - 5843") + ); + expect(rows[0].po_line_number).toEqual(new TextTableValue("")); + expect(rows[0].purchase_order_number).toEqual(new TextTableValue("")); + expect(rows[0].purchase_price_per_unit).toEqual(new FloatTableValue(59)); + expect(rows[0].quantity).toEqual(new IntegerTableValue(1)); + expect(rows[0].release_date).toEqual(new DateTableValue("")); + expect(rows[0].seller).toEqual(new TextTableValue("Amazon.com")); + expect(rows[0].seller_credentials).toEqual(new TextTableValue("")); + expect(rows[0].shipment_date).toEqual(new DateTableValue("01/13/22")); + expect(rows[0].shipping_address_city).toEqual( + new TextTableValue("BLOOMINGTON") + ); + expect(rows[0].shipping_address_name).toEqual(new TextTableValue("Z Hay")); + expect(rows[0].shipping_address_state).toEqual(new TextTableValue("CO")); + expect(rows[0].shipping_address_street_1).toEqual( + new TextTableValue("123 Street APT B") + ); + expect(rows[0].shipping_address_street_2).toEqual(new TextTableValue("")); + expect(rows[0].shipping_address_zip).toEqual( + new TextTableValue("80305-6267") + ); + expect(rows[0].tax_exemption_applied).toEqual(new BoolTableValue(false)); + expect(rows[0].tax_exemption_type).toEqual(new TextTableValue("")); + expect(rows[0].title).toEqual( + new TextTableValue("Apple 67W USB-C Power Adapter") + ); + expect(rows[0].unspsc_code).toEqual(new TextTableValue("26111704")); + expect(rows[0].website).toEqual(new UrlTableValue("Amazon.com")); + }); + + test("it should load a file with no orders", async () => { + const data = await loadTestFileAsText( + "/csv/amazon_orders/01-Jan-2010_to_08-Oct-2010.csv" + ); + + ordersCsv.fileContents = data; + + await ordersCsv.process(); + const { rows } = ordersCsv.table; + + expect(rows.length).toBe(0); }); }); From 6e2672830560468dcecf53c8a63822724b5622c5 Mon Sep 17 00:00:00 2001 From: Roman Meszaros Date: Thu, 27 Oct 2022 14:58:05 +0200 Subject: [PATCH 3/3] REFACTOR CsvExtractor returns [] on error --- src/extractor/csvExtractor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extractor/csvExtractor.ts b/src/extractor/csvExtractor.ts index a9935d2..e0ac5ec 100644 --- a/src/extractor/csvExtractor.ts +++ b/src/extractor/csvExtractor.ts @@ -15,7 +15,7 @@ export class CsvExtractor extends FileExtractor { text: string, options?: Partial ) { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { const csvConfig: csvParser.ParseWorkerConfig = { // Default CSV parsing options header: true, @@ -27,7 +27,7 @@ export class CsvExtractor extends FileExtractor { `Error parsing ${this.zipEntry?.data?.filename}`, results.errors ); - reject(results.errors); + resolve([]); } resolve(caseInsensitiveWrapper(results.data) as []); },