From af2ac90a42327da9e309e819c8a68e8ffd02ae11 Mon Sep 17 00:00:00 2001 From: Will Munn Date: Sat, 12 Apr 2025 00:34:16 +0200 Subject: [PATCH 01/12] Support for batch synonym input via a csv file --- frontend/package.json | 1 + frontend/src/app/app.module.ts | 2 +- .../rules-search/rules-search.component.html | 44 +++++++++++++++ .../rules-search/rules-search.component.ts | 53 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index b0e9bd78..ac32490f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,6 +27,7 @@ "angular2-multiselect-dropdown": "^4.6.6", "angular2-toaster": "^11.0.1", "bootstrap": "^4.5.0", + "papaparse": "^5.5.2", "rxjs": "~6.6.0", "tslib": "^2.0.0", "zone.js": "~0.10.2" diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 7c7b44d9..7eea0060 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -100,7 +100,7 @@ import { SuggestedFieldsComponent, SuggestedFieldsCreateComponent, SuggestedFieldsListComponent, - PreviewLinkComponent + PreviewLinkComponent, ], providers: [ CommonsService, diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html index 8cf49374..3ba28d52 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html @@ -45,6 +45,13 @@ > New + @@ -77,3 +84,40 @@ + +
+

+ Your csv must be in the following format (do not include the header row) +

+ + + + + + + + + + + + + + + +
search termsynonymcomment
bootshoeRequested by footware
+
+
+
+ + +
+
+ diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index a617bc33..fe2c2401 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -14,6 +14,7 @@ import { SpellingsService, TagsService } from '../../../services'; +import Papa from 'papaparse'; import {InputTag, ListItem} from '../../../models'; @Component({ @@ -65,6 +66,48 @@ export class RulesSearchComponent implements OnChanges { }); } + openFileModal(): void { + this.modalService.open('file-import'); + } + + fileSelect(event: Event): void { + const element = event.currentTarget as HTMLInputElement; + const file = element?.files?.[0]; + Papa.parse(file, { + complete: (results) => { + const ruleCreations = results + .data.filter(row => row.length === 3) + .map(row => { + console.log({row}); + this.ruleManagementService + .addNewRuleItem(this.currentSolrIndexId, row[0], []) + .then(ruleId => { + console.log(ruleId); + this.ruleManagementService.updateSearchInput({ + id: ruleId.returnId, + term: row[0], + synonymRules: [{term: row[1], isActive: true, synonymType: 0, id: this.randomUUID()}], + isActive: true, + redirectRules: [], + deleteRules: [], + filterRules: [], + tags: [], + upDownRules: [], + comment: row[2], + term: row[0] + }) + .then(ruleId => + this.refreshAndSelectListItemById.emit(ruleId.returnId) + ); + }) + }); + Promise.all(ruleCreations).then( + () => this.modalService.close('file-import') + ); + } + }); + } + createNewSpellingItem() { if (this.currentSolrIndexId) { this.spellingsService @@ -78,6 +121,16 @@ export class RulesSearchComponent implements OnChanges { } } + private randomUUID() { + /* eslint-disable */ + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = (Math.random() * 16) | 0, + v = c == 'x' ? r : (r & 0x3) | 0x8 + return v.toString(16) + }) + /* eslint-enable */ + } + createNewRuleItem() { if (this.currentSolrIndexId) { const tags = this.appliedTagFilter ? [this.appliedTagFilter.id] : []; From 1e21811d2e8fc86befcb6782e6c42b3b9f04859b Mon Sep 17 00:00:00 2001 From: Will Munn Date: Sun, 13 Apr 2025 18:23:04 +0200 Subject: [PATCH 02/12] Extract logic for csv parsing into separate file --- .../rule-management.component.ts | 14 +---- .../rules-search/rules-search.component.ts | 60 +++++++------------ frontend/src/app/lib/csv.ts | 37 ++++++++++++ frontend/src/app/lib/uuid.ts | 10 ++++ frontend/src/app/services/commons.service.ts | 8 +-- 5 files changed, 74 insertions(+), 55 deletions(-) create mode 100644 frontend/src/app/lib/csv.ts create mode 100644 frontend/src/app/lib/uuid.ts diff --git a/frontend/src/app/components/details/rule-management/rule-management.component.ts b/frontend/src/app/components/details/rule-management/rule-management.component.ts index 5cb6dd83..544e59af 100755 --- a/frontend/src/app/components/details/rule-management/rule-management.component.ts +++ b/frontend/src/app/components/details/rule-management/rule-management.component.ts @@ -27,6 +27,7 @@ import { UpDownRule, PreviewSection } from '../../../models'; +import {randomUUID} from '../../../lib/uuid'; import { CommonsService, FeatureToggleService, @@ -284,7 +285,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC console.log('In SearchInputDetailComponent :: addNewSynonym'); const emptySynonymRule: SynonymRule = { - id: this.randomUUID(), + id: randomUUID(), synonymType: 0, term: '', isActive: true @@ -560,17 +561,6 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC return idxMinimumDistance; } - // taken from https://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript - private randomUUID() { - /* eslint-disable */ - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - const r = (Math.random() * 16) | 0, - v = c == 'x' ? r : (r & 0x3) | 0x8 - return v.toString(16) - }) - /* eslint-enable */ - } - private updateSelectedTagsInModel() { if (this.detailSearchInput) { this.detailSearchInput.tags = this.selectedTags; diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index fe2c2401..37514d7d 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -16,6 +16,7 @@ import { } from '../../../services'; import Papa from 'papaparse'; import {InputTag, ListItem} from '../../../models'; +import {rowsToSearchInputs} from '../../../lib/csv'; @Component({ selector: 'app-smui-rules-search', @@ -75,34 +76,29 @@ export class RulesSearchComponent implements OnChanges { const file = element?.files?.[0]; Papa.parse(file, { complete: (results) => { - const ruleCreations = results - .data.filter(row => row.length === 3) - .map(row => { - console.log({row}); - this.ruleManagementService - .addNewRuleItem(this.currentSolrIndexId, row[0], []) - .then(ruleId => { - console.log(ruleId); - this.ruleManagementService.updateSearchInput({ - id: ruleId.returnId, - term: row[0], - synonymRules: [{term: row[1], isActive: true, synonymType: 0, id: this.randomUUID()}], - isActive: true, - redirectRules: [], - deleteRules: [], - filterRules: [], - tags: [], - upDownRules: [], - comment: row[2], - term: row[0] - }) - .then(ruleId => - this.refreshAndSelectListItemById.emit(ruleId.returnId) - ); - }) + const searchInputs = rowsToSearchInputs(results.data); + const ruleCreations = searchInputs + .map(searchInput => { + return this.ruleManagementService.addNewRuleItem(this.currentSolrIndexId, searchInput.term, []) + .then(inputId => { + searchInput.id = inputId.returnId; + return this.ruleManagementService.updateSearchInput(searchInput) + }) + .then(() => new Promise((resolve, reject) => setTimeout(resolve, 100))); }); - Promise.all(ruleCreations).then( - () => this.modalService.close('file-import') + + // save all rules syncronously (there seems to be an issue with saving rules in parallel) + ruleCreations.reduce( + (promiseChain, creation) => promiseChain.then(creation), + Promise.resolve() + ).then( + () => { + this.modalService.close('file-import') + // wait for all rules to be persisted before refreshing list to ensure correct state + setTimeout(() => { + this.refreshAndSelectListItemById.emit(searchInputs[0].id); + }, 1000); + } ); } }); @@ -121,16 +117,6 @@ export class RulesSearchComponent implements OnChanges { } } - private randomUUID() { - /* eslint-disable */ - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - const r = (Math.random() * 16) | 0, - v = c == 'x' ? r : (r & 0x3) | 0x8 - return v.toString(16) - }) - /* eslint-enable */ - } - createNewRuleItem() { if (this.currentSolrIndexId) { const tags = this.appliedTagFilter ? [this.appliedTagFilter.id] : []; diff --git a/frontend/src/app/lib/csv.ts b/frontend/src/app/lib/csv.ts new file mode 100644 index 00000000..88252664 --- /dev/null +++ b/frontend/src/app/lib/csv.ts @@ -0,0 +1,37 @@ +import {randomUUID} from './uuid'; + +function makeActiveSynonymTerm(synonymTerm) { + return { + term: synonymTerm, + isActive: true, + synonymType: 0, + id: randomUUID() + }; +} + +export function rowsToSearchInputs(rows: string[][]): SearchInput[] { + return rows + .filter(row => row.length === 3) + .reduce((searchInputs, row) => { + console.log(row); + const [term, synonymTerm, comment] = row; + const searchInput = searchInputs.find(searchInput => searchInput.term === term); + if (!searchInput) { + searchInputs = searchInputs.concat({ + id: randomUUID(), + term, + synonymRules: [makeActiveSynonymTerm(synonymTerm)], + isActive: true, + redirectRules: [], + deleteRules: [], + filterRules: [], + tags: [], + upDownRules: [], + comment, + }) + } else { + searchInput.synonymRules.push(makeActiveSynonymTerm(synonymTerm)) + } + return searchInputs; + }, []); +} diff --git a/frontend/src/app/lib/uuid.ts b/frontend/src/app/lib/uuid.ts new file mode 100644 index 00000000..93fd69fc --- /dev/null +++ b/frontend/src/app/lib/uuid.ts @@ -0,0 +1,10 @@ + +// taken from https://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript +export function randomUUID(): string { + /* eslint-disable */ + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + /* eslint-enable */ +} diff --git a/frontend/src/app/services/commons.service.ts b/frontend/src/app/services/commons.service.ts index c7559e64..5a7715a1 100644 --- a/frontend/src/app/services/commons.service.ts +++ b/frontend/src/app/services/commons.service.ts @@ -1,15 +1,11 @@ import { Injectable, SimpleChanges } from '@angular/core'; +import {randomUUID} from '../lib/uuid'; @Injectable() export class CommonsService { generateUUID(): string { - /* eslint-disable */ - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); - return v.toString(16); - }); - /* eslint-enable */ + return randomUUID(); } isDirty(obj: any, origObj: string): boolean { From 53e6940ee0312567e16645f5323bcc3e3ac11876 Mon Sep 17 00:00:00 2001 From: Will Munn Date: Sun, 13 Apr 2025 22:02:06 +0200 Subject: [PATCH 03/12] Remove sleeps --- .../rules-search/rules-search.component.ts | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index 37514d7d..b380d82e 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -78,26 +78,20 @@ export class RulesSearchComponent implements OnChanges { complete: (results) => { const searchInputs = rowsToSearchInputs(results.data); const ruleCreations = searchInputs - .map(searchInput => { - return this.ruleManagementService.addNewRuleItem(this.currentSolrIndexId, searchInput.term, []) - .then(inputId => { - searchInput.id = inputId.returnId; - return this.ruleManagementService.updateSearchInput(searchInput) - }) - .then(() => new Promise((resolve, reject) => setTimeout(resolve, 100))); - }); - - // save all rules syncronously (there seems to be an issue with saving rules in parallel) - ruleCreations.reduce( - (promiseChain, creation) => promiseChain.then(creation), - Promise.resolve() - ).then( + .reduce((chain, searchInput) => { + return chain + .then(() => { + return this.ruleManagementService.addNewRuleItem(this.currentSolrIndexId, searchInput.term, []) + .then(inputId => { + searchInput.id = inputId.returnId; + return this.ruleManagementService.updateSearchInput(searchInput) + }); + }); + }, Promise.resolve()); + ruleCreations.then( () => { this.modalService.close('file-import') - // wait for all rules to be persisted before refreshing list to ensure correct state - setTimeout(() => { - this.refreshAndSelectListItemById.emit(searchInputs[0].id); - }, 1000); + this.refreshAndSelectListItemById.emit(searchInputs[0].id); } ); } From e69d733ad53b726dd8f52d2c0fcc47f8376478db Mon Sep 17 00:00:00 2001 From: Will Munn Date: Sun, 13 Apr 2025 22:21:18 +0200 Subject: [PATCH 04/12] Basic error handling, styling tidyup accept only csv files --- .../rules-search/rules-search.component.css | 4 ++++ .../rules-search/rules-search.component.html | 3 ++- .../rules-search/rules-search.component.ts | 17 +++++++++++------ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.css b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.css index 8d957cca..f16c4954 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.css +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.css @@ -10,3 +10,7 @@ .create-button { margin-left: 0.5rem; } + +.full-width { + width: 100%; +} diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html index 3ba28d52..6b346122 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html @@ -109,10 +109,11 @@ -
+
{ - this.modalService.close('file-import') - this.refreshAndSelectListItemById.emit(searchInputs[0].id); - } - ); + ruleCreations + .then( + () => { + this.modalService.close('file-import') + this.refreshAndSelectListItemById.emit(searchInputs[0].id); + } + ) + .error(err => this.showErrorMsg.emit(err.message)); + }, + error: (err, file, inputElem, reason) => { + this.showErrorMsg.emit(err); } }); } From 4ebeb6e7a3789a42fa0cf996079b3a43eb9e294a Mon Sep 17 00:00:00 2001 From: Will Munn Date: Mon, 14 Apr 2025 08:14:04 +0200 Subject: [PATCH 05/12] Add unit tests for csv parsing --- frontend/src/app/lib/csv.spec.ts | 50 ++++++++++++++++++++++++++++++++ frontend/src/app/lib/csv.ts | 5 ++-- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 frontend/src/app/lib/csv.spec.ts diff --git a/frontend/src/app/lib/csv.spec.ts b/frontend/src/app/lib/csv.spec.ts new file mode 100644 index 00000000..c5d6cfbc --- /dev/null +++ b/frontend/src/app/lib/csv.spec.ts @@ -0,0 +1,50 @@ +import {rowsToSearchInputs} from './csv'; + +describe('rowsToSearchInputs', () => { + it('parses no inputs when there are no rows', () => { + expect(rowsToSearchInputs([])).toHaveSize(0); + }); + + it('ignores rows with less than 3 columns', () => { + expect(rowsToSearchInputs([['abc', '123']])).toHaveSize(0); + }); + + it('ignores rows with more than 3 columns', () => { + expect(rowsToSearchInputs([['abc', '123', 'def', '456']])).toHaveSize(0); + }); + + it('parses rows with 3 columns', () => { + const inputs = rowsToSearchInputs([ + ['term', 'synonym', 'comment'] + ]); + expect(inputs).toHaveSize(1); + expect(inputs[0].id).toBeInstanceOf(String); + expect(inputs[0].term).toEqual('term'); + expect(inputs[0].redirectRules).toEqual([]); + expect(inputs[0].filterRules).toEqual([]); + expect(inputs[0].tags).toEqual([]); + expect(inputs[0].upDownRules).toEqual([]); + expect(inputs[0].synonymRules).toHaveSize(1); + expect(inputs[0].synonymRules[0].term).toEqual('synonym'); + expect(inputs[0].synonymRules[0].isActive).toEqual(true); + expect(inputs[0].synonymRules[0].synonymType).toEqual(0); + expect(inputs[0].synonymRules[0].id).toBeInstanceOf(String); + }); + + it('groups rows with the same search term', () => { + const inputs = rowsToSearchInputs([ + ['term', 'synonym', 'comment'], + ['term', 'another synonym', 'comment'], + ]); + expect(inputs).toHaveSize(1); + expect(inputs[0].synonymRules).toHaveSize(2); + }); + + it('creates one SearchInput per term', () => { + const inputs = rowsToSearchInputs([ + ['term', 'synonym', 'comment'], + ['term2', 'another synonym', 'comment'], + ]); + expect(inputs).toHaveSize(2); + }); +}); diff --git a/frontend/src/app/lib/csv.ts b/frontend/src/app/lib/csv.ts index 88252664..b3ef8ddc 100644 --- a/frontend/src/app/lib/csv.ts +++ b/frontend/src/app/lib/csv.ts @@ -1,6 +1,7 @@ +import {SynonymRule, SearchInput} from '../models'; import {randomUUID} from './uuid'; -function makeActiveSynonymTerm(synonymTerm) { +function makeActiveSynonymTerm(synonymTerm: string): SynonymRule { return { term: synonymTerm, isActive: true, @@ -33,5 +34,5 @@ export function rowsToSearchInputs(rows: string[][]): SearchInput[] { searchInput.synonymRules.push(makeActiveSynonymTerm(synonymTerm)) } return searchInputs; - }, []); + }, [] as SearchInput[]); } From 1167cb580674a49d518ea490db93d30371a75a22 Mon Sep 17 00:00:00 2001 From: Will Munn Date: Mon, 14 Apr 2025 08:21:18 +0200 Subject: [PATCH 06/12] Extract constant for file modal name --- .../rules-panel/rules-search/rules-search.component.ts | 5 +++-- frontend/src/app/lib/csv.ts | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index d1731207..e5487cee 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -17,6 +17,7 @@ import { import Papa from 'papaparse'; import {InputTag, ListItem} from '../../../models'; import {rowsToSearchInputs} from '../../../lib/csv'; +const fileImportModal = 'file-import'; @Component({ selector: 'app-smui-rules-search', @@ -68,7 +69,7 @@ export class RulesSearchComponent implements OnChanges { } openFileModal(): void { - this.modalService.open('file-import'); + this.modalService.open(fileImportModal); } fileSelect(event: Event): void { @@ -91,7 +92,7 @@ export class RulesSearchComponent implements OnChanges { ruleCreations .then( () => { - this.modalService.close('file-import') + this.modalService.close(fileImportModal) this.refreshAndSelectListItemById.emit(searchInputs[0].id); } ) diff --git a/frontend/src/app/lib/csv.ts b/frontend/src/app/lib/csv.ts index b3ef8ddc..f9392b5c 100644 --- a/frontend/src/app/lib/csv.ts +++ b/frontend/src/app/lib/csv.ts @@ -14,7 +14,6 @@ export function rowsToSearchInputs(rows: string[][]): SearchInput[] { return rows .filter(row => row.length === 3) .reduce((searchInputs, row) => { - console.log(row); const [term, synonymTerm, comment] = row; const searchInput = searchInputs.find(searchInput => searchInput.term === term); if (!searchInput) { From 9308a6f059096261ac64c4241dfdf3fccfe44d6a Mon Sep 17 00:00:00 2001 From: Will Munn Date: Mon, 14 Apr 2025 08:23:15 +0200 Subject: [PATCH 07/12] Handle empty csvs --- .../rules-panel/rules-search/rules-search.component.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index e5487cee..3993787d 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -93,7 +93,9 @@ export class RulesSearchComponent implements OnChanges { .then( () => { this.modalService.close(fileImportModal) - this.refreshAndSelectListItemById.emit(searchInputs[0].id); + if (searchInputs.length > 0) { + this.refreshAndSelectListItemById.emit(searchInputs[0].id); + } } ) .error(err => this.showErrorMsg.emit(err.message)); From adbe6f92b8234c5d8b50c710a53d8e3a27f8795b Mon Sep 17 00:00:00 2001 From: Will Munn Date: Mon, 14 Apr 2025 20:47:41 +0200 Subject: [PATCH 08/12] Fix type errors --- .../rule-management.component.ts | 8 +- .../rules-search/rules-search.component.ts | 73 ++++++++++--------- .../src/app/services/csv-import.service.ts | 12 +++ frontend/src/app/services/index.ts | 1 + 4 files changed, 57 insertions(+), 37 deletions(-) create mode 100644 frontend/src/app/services/csv-import.service.ts diff --git a/frontend/src/app/components/details/rule-management/rule-management.component.ts b/frontend/src/app/components/details/rule-management/rule-management.component.ts index 544e59af..4e4003a6 100755 --- a/frontend/src/app/components/details/rule-management/rule-management.component.ts +++ b/frontend/src/app/components/details/rule-management/rule-management.component.ts @@ -309,7 +309,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC console.log('In SearchInputDetailComponent :: addNewUpDownRule'); const emptyUpDownRule: UpDownRule = { - id: this.randomUUID(), + id: randomUUID(), term: '', isActive: true }; @@ -341,7 +341,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC console.log('In SearchInputDetailComponent :: addNewFilterRule'); const emptyFilterRule: FilterRule = { - id: this.randomUUID(), + id: randomUUID(), term: '', isActive: true }; @@ -370,7 +370,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC console.log('In SearchInputDetailComponent :: addNewDeleteRule'); const emptyDeleteRule: DeleteRule = { - id: this.randomUUID(), + id: randomUUID(), term: '', isActive: true }; @@ -394,7 +394,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC console.log('In SearchInputDetailComponent :: addNewRedirectRule'); const emptyRedirectRule: RedirectRule = { - id: this.randomUUID(), + id: randomUUID(), target: '', isActive: true }; diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index 3993787d..315871bc 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -12,10 +12,11 @@ import { ModalService, RuleManagementService, SpellingsService, - TagsService + TagsService, + CSVImportService } from '../../../services'; -import Papa from 'papaparse'; -import {InputTag, ListItem} from '../../../models'; +import {parse} from 'papaparse'; +import {InputTag, ListItem, ApiResult} from '../../../models'; import {rowsToSearchInputs} from '../../../lib/csv'; const fileImportModal = 'file-import'; @@ -46,7 +47,8 @@ export class RulesSearchComponent implements OnChanges { private ruleManagementService: RuleManagementService, private spellingsService: SpellingsService, private tagsService: TagsService, - private modalService: ModalService + private modalService: ModalService, + private csvImportService: CSVImportService ) {} ngOnChanges(changes: SimpleChanges): void { @@ -74,36 +76,41 @@ export class RulesSearchComponent implements OnChanges { fileSelect(event: Event): void { const element = event.currentTarget as HTMLInputElement; - const file = element?.files?.[0]; - Papa.parse(file, { - complete: (results) => { - const searchInputs = rowsToSearchInputs(results.data); - const ruleCreations = searchInputs - .reduce((chain, searchInput) => { - return chain - .then(() => { - return this.ruleManagementService.addNewRuleItem(this.currentSolrIndexId, searchInput.term, []) - .then(inputId => { - searchInput.id = inputId.returnId; - return this.ruleManagementService.updateSearchInput(searchInput) - }); - }); - }, Promise.resolve()); - ruleCreations - .then( - () => { - this.modalService.close(fileImportModal) - if (searchInputs.length > 0) { - this.refreshAndSelectListItemById.emit(searchInputs[0].id); + if (element?.files?.length && this.currentSolrIndexId) { + const files: FileList = element?.files; + const file = element?.files?.[0]; + parse(file, { + complete: (results: {data: string[][]}) => { + this.csvImportService.import(); + const searchInputs = rowsToSearchInputs(results.data); + const ruleCreations: Promise = searchInputs + .reduce((chain: Promise, searchInput): Promise => { + return chain + .then(() => { + return this.ruleManagementService.addNewRuleItem(this.currentSolrIndexId as string, searchInput.term, []) + .then(inputId => { + searchInput.id = inputId.returnId; + return this.ruleManagementService.updateSearchInput(searchInput) + }); + }); + }, Promise.resolve(null)); + ruleCreations + .then( + () => { + this.modalService.close(fileImportModal) + if (searchInputs.length > 0) { + this.refreshAndSelectListItemById.emit(searchInputs[0].id); + } } - } - ) - .error(err => this.showErrorMsg.emit(err.message)); - }, - error: (err, file, inputElem, reason) => { - this.showErrorMsg.emit(err); - } - }); + ) + .catch(err => this.showErrorMsg.emit(err.message)); + }, + error: (err: Error) => { + this.showErrorMsg.emit(err?.message); + } + + }); + } } createNewSpellingItem() { diff --git a/frontend/src/app/services/csv-import.service.ts b/frontend/src/app/services/csv-import.service.ts new file mode 100644 index 00000000..c9117cc3 --- /dev/null +++ b/frontend/src/app/services/csv-import.service.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@angular/core'; + +@Injectable({ + providedIn: 'root' +}) +export class CSVImportService { + constructor() {} + + import() { + console.log('hello'); + } +} diff --git a/frontend/src/app/services/index.ts b/frontend/src/app/services/index.ts index ec789ae3..2365e212 100644 --- a/frontend/src/app/services/index.ts +++ b/frontend/src/app/services/index.ts @@ -11,3 +11,4 @@ export * from './config.service'; export * from './modal.service'; export * from './deployment-detailed-info.service'; export * from './preview-link.service'; +export * from './csv-import.service'; From f4ee2a52064c6e5d9bfcb3be2fa83ec7facb0f53 Mon Sep 17 00:00:00 2001 From: Will Munn Date: Mon, 14 Apr 2025 21:09:56 +0200 Subject: [PATCH 09/12] Extract csv import service --- .../rules-search/rules-search.component.ts | 41 ++++--------------- .../src/app/services/csv-import.service.ts | 32 +++++++++++++-- frontend/src/papaparse.d.ts | 1 + 3 files changed, 38 insertions(+), 36 deletions(-) create mode 100644 frontend/src/papaparse.d.ts diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index 315871bc..748dcfa0 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -15,9 +15,7 @@ import { TagsService, CSVImportService } from '../../../services'; -import {parse} from 'papaparse'; import {InputTag, ListItem, ApiResult} from '../../../models'; -import {rowsToSearchInputs} from '../../../lib/csv'; const fileImportModal = 'file-import'; @Component({ @@ -79,37 +77,14 @@ export class RulesSearchComponent implements OnChanges { if (element?.files?.length && this.currentSolrIndexId) { const files: FileList = element?.files; const file = element?.files?.[0]; - parse(file, { - complete: (results: {data: string[][]}) => { - this.csvImportService.import(); - const searchInputs = rowsToSearchInputs(results.data); - const ruleCreations: Promise = searchInputs - .reduce((chain: Promise, searchInput): Promise => { - return chain - .then(() => { - return this.ruleManagementService.addNewRuleItem(this.currentSolrIndexId as string, searchInput.term, []) - .then(inputId => { - searchInput.id = inputId.returnId; - return this.ruleManagementService.updateSearchInput(searchInput) - }); - }); - }, Promise.resolve(null)); - ruleCreations - .then( - () => { - this.modalService.close(fileImportModal) - if (searchInputs.length > 0) { - this.refreshAndSelectListItemById.emit(searchInputs[0].id); - } - } - ) - .catch(err => this.showErrorMsg.emit(err.message)); - }, - error: (err: Error) => { - this.showErrorMsg.emit(err?.message); - } - - }); + const ruleCreations = this.csvImportService.import(file, this.currentSolrIndexId); + ruleCreations + .then( + () => { + this.modalService.close(fileImportModal) + this.refreshAndSelectListItemById.emit(); + } + ).catch(err => this.showErrorMsg.emit(err.message)); } } diff --git a/frontend/src/app/services/csv-import.service.ts b/frontend/src/app/services/csv-import.service.ts index c9117cc3..c19740b9 100644 --- a/frontend/src/app/services/csv-import.service.ts +++ b/frontend/src/app/services/csv-import.service.ts @@ -1,12 +1,38 @@ +import {parse} from 'papaparse'; import { Injectable } from '@angular/core'; +import {ApiResult} from '../models'; +import {rowsToSearchInputs} from '../lib/csv'; +import {RuleManagementService, ModalService} from '.'; +const fileImportModal = 'file-import'; @Injectable({ providedIn: 'root' }) export class CSVImportService { - constructor() {} + constructor(private ruleManagementService: RuleManagementService, private modalService: ModalService) {} - import() { - console.log('hello'); + import(file: File, indexId: string): Promise { + return new Promise((resolve, reject) => { + parse(file, { + complete: (results: {data: string[][]}) => { + const searchInputs = rowsToSearchInputs(results.data); + const ruleCreations: Promise = searchInputs + .reduce((chain: Promise, searchInput): Promise => { + return chain + .then(() => { + return this.ruleManagementService.addNewRuleItem(indexId, searchInput.term, []) + .then(inputId => { + searchInput.id = inputId.returnId; + return this.ruleManagementService.updateSearchInput(searchInput) + }); + }); + }, Promise.resolve(null)); + resolve(ruleCreations); + }, + error: (err: Error) => { + reject(err); + } + }); + }); } } diff --git a/frontend/src/papaparse.d.ts b/frontend/src/papaparse.d.ts new file mode 100644 index 00000000..f5e5feef --- /dev/null +++ b/frontend/src/papaparse.d.ts @@ -0,0 +1 @@ +declare module 'papaparse'; From 3285639b64d319b151385dba3fedc495a0579adc Mon Sep 17 00:00:00 2001 From: Will Munn Date: Mon, 14 Apr 2025 22:07:01 +0200 Subject: [PATCH 10/12] Add progress bar --- .../rules-search/rules-search.component.html | 9 ++++++++ .../rules-search/rules-search.component.ts | 22 +++++++++++++++---- .../src/app/services/csv-import.service.ts | 7 +++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html index 6b346122..94f510cb 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html @@ -89,6 +89,15 @@ title="Import rules from CSV" >
+
+
+

Your csv must be in the following format (do not include the header row)

diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts index 748dcfa0..e5e4430c 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts @@ -4,7 +4,8 @@ import { Output, EventEmitter, OnChanges, - SimpleChanges + SimpleChanges, + ChangeDetectorRef } from '@angular/core'; import { @@ -37,6 +38,7 @@ export class RulesSearchComponent implements OnChanges { @Output() showErrorMsg: EventEmitter = new EventEmitter(); allTags: InputTag[] = []; + progress: number = 0; readonly isTaggingActive = this.featureToggleService.isRuleTaggingActive(); private readonly isSpellingActive = this.featureToggleService.getSyncToggleActivateSpelling(); @@ -46,7 +48,8 @@ export class RulesSearchComponent implements OnChanges { private spellingsService: SpellingsService, private tagsService: TagsService, private modalService: ModalService, - private csvImportService: CSVImportService + private csvImportService: CSVImportService, + private cdr: ChangeDetectorRef ) {} ngOnChanges(changes: SimpleChanges): void { @@ -77,14 +80,25 @@ export class RulesSearchComponent implements OnChanges { if (element?.files?.length && this.currentSolrIndexId) { const files: FileList = element?.files; const file = element?.files?.[0]; - const ruleCreations = this.csvImportService.import(file, this.currentSolrIndexId); + const ruleCreations = this.csvImportService.import( + file, + this.currentSolrIndexId, + (percentage) => { + this.progress = percentage; + this.cdr.detectChanges(); + } + ); ruleCreations .then( () => { + this.progress = 0; this.modalService.close(fileImportModal) this.refreshAndSelectListItemById.emit(); } - ).catch(err => this.showErrorMsg.emit(err.message)); + ).catch(err => { + this.showErrorMsg.emit(err.message) + this.progress = 0; + }); } } diff --git a/frontend/src/app/services/csv-import.service.ts b/frontend/src/app/services/csv-import.service.ts index c19740b9..0a836f26 100644 --- a/frontend/src/app/services/csv-import.service.ts +++ b/frontend/src/app/services/csv-import.service.ts @@ -11,17 +11,22 @@ const fileImportModal = 'file-import'; export class CSVImportService { constructor(private ruleManagementService: RuleManagementService, private modalService: ModalService) {} - import(file: File, indexId: string): Promise { + import(file: File, indexId: string, progress: () => void): Promise { return new Promise((resolve, reject) => { parse(file, { complete: (results: {data: string[][]}) => { const searchInputs = rowsToSearchInputs(results.data); + let i = 1; const ruleCreations: Promise = searchInputs .reduce((chain: Promise, searchInput): Promise => { return chain .then(() => { return this.ruleManagementService.addNewRuleItem(indexId, searchInput.term, []) .then(inputId => { + const onePercent = 100 / (searchInputs.length); + + progress(onePercent * i); + i ++; searchInput.id = inputId.returnId; return this.ruleManagementService.updateSearchInput(searchInput) }); From 7e71601223cdaca5052b9300257853c72d1dd2f2 Mon Sep 17 00:00:00 2001 From: Will Munn Date: Tue, 15 Apr 2025 10:06:52 +0200 Subject: [PATCH 11/12] Styling fixes for progress bar --- .../rules-panel/rules-search/rules-search.component.html | 2 +- .../rules-panel/rules-search/rules-search.component.ts | 2 +- frontend/src/app/services/csv-import.service.ts | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html index 94f510cb..8c546b3d 100755 --- a/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html +++ b/frontend/src/app/components/rules-panel/rules-search/rules-search.component.html @@ -89,7 +89,7 @@ title="Import rules from CSV" >
-
+
{ + (percentage: number) => { this.progress = percentage; this.cdr.detectChanges(); } diff --git a/frontend/src/app/services/csv-import.service.ts b/frontend/src/app/services/csv-import.service.ts index 0a836f26..15e00e23 100644 --- a/frontend/src/app/services/csv-import.service.ts +++ b/frontend/src/app/services/csv-import.service.ts @@ -2,7 +2,8 @@ import {parse} from 'papaparse'; import { Injectable } from '@angular/core'; import {ApiResult} from '../models'; import {rowsToSearchInputs} from '../lib/csv'; -import {RuleManagementService, ModalService} from '.'; +import {RuleManagementService} from './rule-management.service' +import {ModalService} from './modal.service'; const fileImportModal = 'file-import'; @Injectable({ @@ -11,7 +12,7 @@ const fileImportModal = 'file-import'; export class CSVImportService { constructor(private ruleManagementService: RuleManagementService, private modalService: ModalService) {} - import(file: File, indexId: string, progress: () => void): Promise { + import(file: File, indexId: string, progress: (percentage: number) => void): Promise { return new Promise((resolve, reject) => { parse(file, { complete: (results: {data: string[][]}) => { From d31c021e624903912af835ba4e5ac00d349fafbb Mon Sep 17 00:00:00 2001 From: Will Munn Date: Fri, 18 Apr 2025 15:15:38 +0200 Subject: [PATCH 12/12] Add tests for csv import service --- .../app/services/csv-import.service.spec.ts | 54 +++++++++++++++++++ .../src/app/services/csv-import.service.ts | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 frontend/src/app/services/csv-import.service.spec.ts diff --git a/frontend/src/app/services/csv-import.service.spec.ts b/frontend/src/app/services/csv-import.service.spec.ts new file mode 100644 index 00000000..830e2f18 --- /dev/null +++ b/frontend/src/app/services/csv-import.service.spec.ts @@ -0,0 +1,54 @@ +import { CSVImportService } from './csv-import.service'; +import {RuleManagementService} from './rule-management.service' + +describe('CSVImportService', () => { + let csvImportService: CSVImportService; + + it('saves rule items', (done) => { + const csv = "term,synonym,comment" as unknown as File; + + csvImportService.import(csv, 'indexId', () => {}).then(x => { + expect(addRuleItem).toHaveBeenCalledOnceWith('indexId', 'term', []); + + const searchInput = updateSearchInput.calls.first().args[0]; + expect(searchInput.id).toEqual('123'); + expect(searchInput.term).toEqual('term'); + expect(searchInput.synonymRules.length).toEqual(1); + expect(searchInput.synonymRules[0].term).toEqual('synonym'); + expect(searchInput.synonymRules[0].isActive).toEqual(true); + expect(searchInput.synonymRules[0].synonymType).toEqual(0); + expect(searchInput.isActive).toEqual(true); + expect(searchInput.comment).toEqual('comment'); + done(); + }); + }); + + it('calls progress for each saved rule item', (done) => { + const ruleManagementService = { + addNewRuleItem() {}, + updateSearchInput() {} + } as unknown as RuleManagementService; + const ctx = {progress: (percentage: number) => {}} + const progress = spyOn(ctx, 'progress'); + const addRuleItem = spyOn( + ruleManagementService, 'addNewRuleItem' + ).and.returnValue( + Promise.resolve({returnId: '123', message: '', result: ''}) + ); + const updateSearchInput = spyOn( + ruleManagementService, 'updateSearchInput' + ).and.returnValue( + Promise.resolve({result: '', message: '', returnId: ''}) + ); + csvImportService = new CSVImportService(ruleManagementService); + const csv = "term,synonym,comment\nterm2,synonym2,comment2" as unknown as File; + + csvImportService.import(csv, 'indexId', ctx.progress).then(x => { + expect(progress).toHaveBeenCalledTimes(2); + expect(progress).toHaveBeenCalledWith(50); + expect(progress).toHaveBeenCalledWith(100); + done(); + }); + }); + +}); diff --git a/frontend/src/app/services/csv-import.service.ts b/frontend/src/app/services/csv-import.service.ts index 15e00e23..442463cc 100644 --- a/frontend/src/app/services/csv-import.service.ts +++ b/frontend/src/app/services/csv-import.service.ts @@ -10,7 +10,7 @@ const fileImportModal = 'file-import'; providedIn: 'root' }) export class CSVImportService { - constructor(private ruleManagementService: RuleManagementService, private modalService: ModalService) {} + constructor(private ruleManagementService: RuleManagementService) {} import(file: File, indexId: string, progress: (percentage: number) => void): Promise { return new Promise((resolve, reject) => {