diff --git a/src/main/java/com/crowdin/cli/client/CrowdinProject.java b/src/main/java/com/crowdin/cli/client/CrowdinProject.java index 464d8417e..16cdd8130 100644 --- a/src/main/java/com/crowdin/cli/client/CrowdinProject.java +++ b/src/main/java/com/crowdin/cli/client/CrowdinProject.java @@ -1,6 +1,7 @@ package com.crowdin.cli.client; import com.crowdin.client.languages.model.Language; +import com.crowdin.client.projectsgroups.model.Project; import com.crowdin.client.projectsgroups.model.Type; import java.util.ArrayList; @@ -17,6 +18,7 @@ public class CrowdinProject { private LanguageMapping languageMapping; private List projectLanguages; private boolean skipUntranslatedFiles; + private com.crowdin.client.projectsgroups.model.Project project; CrowdinProject() { @@ -97,6 +99,14 @@ public List getProjectLanguages(boolean withInContextLang) { } } + public Project getProject() { + return project; + } + + public void setProject(Project project) { + this.project = project; + } + public Optional findLanguageById(String langId) { return this.getProjectLanguages(true) .stream() diff --git a/src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java b/src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java index 0f08f76d5..9e9ef66b1 100644 --- a/src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java +++ b/src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java @@ -88,6 +88,7 @@ private void populateProjectWithStructure(CrowdinProjectFull project, String bra private void populateProjectWithInfo(CrowdinProject project) { com.crowdin.client.projectsgroups.model.Project projectModel = this.getProject(); + project.setProject(projectModel); project.setProjectId(projectModel.getId()); project.setType(projectModel.getType()); project.setSourceLanguageId(projectModel.getSourceLanguageId()); @@ -423,6 +424,25 @@ public List listSourceString(Long fileId, Long branchId, String la .listSourceStrings(this.projectId, builder.limit(limit).offset(offset).build())); } + @Override + @SneakyThrows + public void batchEditSourceStrings(List request) { + Map, ResponseException> errorHandler = new LinkedHashMap<>() {{ + put((code, message) -> message.contains("Someone else is currently editing one of the file"), + new RepeatException("Someone else is currently editing one of the file.")); + put((code, message) -> code.equals("409"), + new RepeatException("Conflict occurred while batch editing source strings. Please try again.")); + }}; + executeRequestWithPossibleRetries( + errorHandler, + () -> this.client.getSourceStringsApi() + .stringBatchOperations(this.projectId, request) + .getData(), + 3, + 3 * 1000 + ); + } + @Override public void deleteSourceString(Long sourceId) { executeRequest(() -> { diff --git a/src/main/java/com/crowdin/cli/client/ProjectClient.java b/src/main/java/com/crowdin/cli/client/ProjectClient.java index 0a5ea8d0a..27b972076 100644 --- a/src/main/java/com/crowdin/cli/client/ProjectClient.java +++ b/src/main/java/com/crowdin/cli/client/ProjectClient.java @@ -104,6 +104,8 @@ default CrowdinProjectFull downloadFullProject() { List listSourceString(Long fileId, Long branchId, String labelIds, String filter, String croql, Long directory, String scope); + void batchEditSourceStrings(List request); + void deleteSourceString(Long id); StringComment commentString(AddStringCommentRequest request); diff --git a/src/main/java/com/crowdin/cli/commands/Actions.java b/src/main/java/com/crowdin/cli/commands/Actions.java index 6bf99a25b..a8e3c5708 100644 --- a/src/main/java/com/crowdin/cli/commands/Actions.java +++ b/src/main/java/com/crowdin/cli/commands/Actions.java @@ -17,6 +17,7 @@ import java.io.File; import java.nio.file.Path; +import java.time.LocalDate; import java.util.List; import java.util.Map; @@ -164,4 +165,43 @@ NewAction preTranslate( NewAction uninstallApp(String id, Boolean force); NewAction installApp(String identifier); + + NewAction contextDownload( + File to, + List filesFilter, + List labelsFilter, + String branchFilter, + String croqlFilter, + LocalDate sinceFilter, + String statusFilter, + String outputFormat, + FilesInterface files, + boolean plainView, + boolean noProgress + ); + + NewAction contextUpload(File file, boolean overwrite, boolean dryRun, boolean plainView, int batchSize); + + NewAction contextReset( + List filesFilter, + List labelsFilter, + String branchFilter, + String croqlFilter, + LocalDate sinceFilter, + boolean dryRun, + int batchSize, + boolean plainView, + boolean noProgress + ); + + NewAction contextStatus( + List filesFilter, + List labelsFilter, + String branchFilter, + String croqlFilter, + LocalDate sinceFilter, + boolean byFile, + boolean plainView, + boolean noProgress + ); } diff --git a/src/main/java/com/crowdin/cli/commands/actions/CliActions.java b/src/main/java/com/crowdin/cli/commands/actions/CliActions.java index 324b8e9bb..c490fb2da 100644 --- a/src/main/java/com/crowdin/cli/commands/actions/CliActions.java +++ b/src/main/java/com/crowdin/cli/commands/actions/CliActions.java @@ -19,6 +19,7 @@ import java.io.File; import java.nio.file.Path; +import java.time.LocalDate; import java.util.List; import java.util.Map; @@ -358,4 +359,24 @@ public NewAction uninstallApp(String id, Boole public NewAction installApp(String identifier) { return new AppInstallAction(identifier); } + + @Override + public NewAction contextDownload(File to, List filesFilter, List labelsFilter, String branchFilter, String croqlFilter, LocalDate sinceFilter, String statusFilter, String outputFormat, FilesInterface files, boolean plainView, boolean noProgress) { + return new ContextDownloadAction(to, filesFilter, labelsFilter, branchFilter, croqlFilter, sinceFilter, statusFilter, outputFormat, files, plainView, noProgress); + } + + @Override + public NewAction contextUpload(File file, boolean overwrite, boolean dryRun, boolean plainView, int batchSize) { + return new ContextUploadAction(file, overwrite, dryRun, plainView, batchSize); + } + + @Override + public NewAction contextReset(List filesFilter, List labelsFilter, String branchFilter, String croqlFilter, LocalDate sinceFilter, boolean dryRun, int batchSize, boolean plainView, boolean noProgress) { + return new ContextResetAction(filesFilter, labelsFilter, branchFilter, croqlFilter, sinceFilter, dryRun, batchSize, plainView, noProgress); + } + + @Override + public NewAction contextStatus(List filesFilter, List labelsFilter, String branchFilter, String croqlFilter, LocalDate sinceFilter, boolean byFile, boolean plainView, boolean noProgress) { + return new ContextStatusAction(filesFilter, labelsFilter, branchFilter, croqlFilter, sinceFilter, byFile, plainView, noProgress); + } } diff --git a/src/main/java/com/crowdin/cli/commands/actions/ContextDownloadAction.java b/src/main/java/com/crowdin/cli/commands/actions/ContextDownloadAction.java new file mode 100644 index 000000000..57c541977 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/actions/ContextDownloadAction.java @@ -0,0 +1,187 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.CrowdinProjectFull; +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.commands.functionality.FilesInterface; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.cli.utils.AiContextUtil; +import com.crowdin.cli.utils.GlobUtil; +import com.crowdin.cli.utils.StringUtil; +import com.crowdin.cli.utils.Utils; +import com.crowdin.cli.utils.console.ConsoleSpinner; +import com.crowdin.client.labels.model.Label; +import com.crowdin.client.projectsgroups.model.Type; +import com.crowdin.client.sourcefiles.model.Branch; +import com.crowdin.client.sourcefiles.model.FileInfo; +import com.crowdin.client.sourcestrings.model.SourceString; +import lombok.AllArgsConstructor; +import org.json.JSONObject; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.*; +import java.util.stream.Collectors; + +import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE; +import static com.crowdin.cli.utils.console.ExecutionStatus.OK; +import static com.crowdin.cli.utils.console.ExecutionStatus.WARNING; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.nonNull; + +@AllArgsConstructor +class ContextDownloadAction implements NewAction { + + private final File to; + private final List filesFilter; + private final List labelsFilter; + private final String branchFilter; + private final String croqlFilter; + private final LocalDate sinceFilter; + private final String statusFilter; + private final String outputFormat; + private final FilesInterface files; + private final boolean plainView; + private final boolean noProgress; + + @Override + public void act(Outputter out, ProjectProperties pb, ProjectClient client) { + + CrowdinProjectFull project = ConsoleSpinner.execute( + out, + "message.spinner.fetching_project_info", + "error.collect_project_info", + this.noProgress, + this.plainView, + () -> client.downloadFullProject(this.branchFilter) + ); + + List existingRecords = to.exists() ? AiContextUtil.readRecords(to) : List.of(); + + boolean isStringsBasedProject = Objects.equals(project.getType(), Type.STRINGS_BASED); + + Long branchId = Optional.ofNullable(project.getBranch()) + .map(Branch::getId) + .orElse(null); + + Map labelsMap = client.listLabels().stream() + .collect(Collectors.toMap(Label::getTitle, Label::getId, (existing, replacement) -> existing)); + + String filterLabelsStr = Optional + .ofNullable(this.labelsFilter) + .filter(list -> !list.isEmpty()) + .map(list -> list.stream() + .filter(labelsMap::containsKey) + .map(label -> labelsMap.get(label).toString()) + .collect(Collectors.joining(","))) + .orElse(null); + + String encodedCroql = nonNull(croqlFilter) ? Utils.encodeURL(croqlFilter) : null; + + List fileIds = null; + + if (!isStringsBasedProject && nonNull(this.filesFilter) && !this.filesFilter.isEmpty()) { + fileIds = project.getFileInfos().stream() + .filter(file -> this.filesFilter.stream().anyMatch(filter -> GlobUtil.matches(filter, file.getPath()))) + .map(FileInfo::getId) + .toList(); + } + + List strings = new ArrayList<>(); + + if (isStringsBasedProject) { + strings = client.listSourceString(null, branchId, filterLabelsStr, null, encodedCroql, null, null); + } else { + if (nonNull(fileIds) && !fileIds.isEmpty()) { + for (Long fileId : fileIds) { + strings.addAll(client.listSourceString(fileId, branchId, filterLabelsStr, null, encodedCroql, null, null)); + } + } else { + strings = client.listSourceString(null, branchId, filterLabelsStr, null, encodedCroql, null, null); + } + } + + strings = strings.stream() + .filter(string -> { + if (sinceFilter == null) { + return true; + } + var createdDate = string.getCreatedAt().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + + return createdDate.isAfter(sinceFilter) || createdDate.isEqual(sinceFilter); + }) + .toList(); + + strings = strings.stream() + .filter(string -> { + if (statusFilter == null) { + return true; + } + + switch (statusFilter) { + case "empty" -> { + return string.getContext() == null || string.getContext().isEmpty(); + } + case "ai" -> { + String aiContextSection = AiContextUtil.getAiContextSection(string.getContext()); + return !aiContextSection.isEmpty(); + } + case "manual" -> { + String manualContext = AiContextUtil.getManualContext(string.getContext()); + return !manualContext.isEmpty(); + } + default -> { + return true; + } + } + }) + .toList(); + + if (strings.isEmpty()) { + out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("message.source_string_list_not_found"))); + return; + } + + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.downloaded_strings"), strings.size()))); + + if (outputFormat.equals("jsonl")) { + String jsonlOutput = strings.stream() + .map(string -> { + String filePath = project.getFileInfos().stream() + .filter(file -> string.getFileId() != null && file.getId().equals(string.getFileId())) + .findFirst() + .map(FileInfo::getPath) + .orElse(""); + + var existingRecord = existingRecords.stream() + .filter(record -> record.getId().equals(string.getId())) + .findFirst(); + + var stringContextRow = new AiContextUtil.StringContextRecord( + string.getId(), + string.getIdentifier(), + StringUtil.getStringText(string), + filePath, + AiContextUtil.getManualContext(string.getContext()), + existingRecord + .map(AiContextUtil.StringContextRecord::getAi_context) + .orElseGet(() -> AiContextUtil.getAiContextSection(string.getContext())) + ); + + return new JSONObject(stringContextRow).toString(); + }) + .collect(Collectors.joining("\n")); + + try { + files.writeToFile(to.toString(), new ByteArrayInputStream(jsonlOutput.getBytes(UTF_8))); + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.saved_strings"), to))); + } catch (IOException e) { + throw new RuntimeException(RESOURCE_BUNDLE.getString("error.write_file"), e); + } + } + } +} diff --git a/src/main/java/com/crowdin/cli/commands/actions/ContextResetAction.java b/src/main/java/com/crowdin/cli/commands/actions/ContextResetAction.java new file mode 100644 index 000000000..e3ee5f112 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/actions/ContextResetAction.java @@ -0,0 +1,148 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.CrowdinProjectFull; +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.cli.utils.AiContextUtil; +import com.crowdin.cli.utils.GlobUtil; +import com.crowdin.cli.utils.Utils; +import com.crowdin.cli.utils.console.ConsoleSpinner; +import com.crowdin.client.core.model.PatchOperation; +import com.crowdin.client.core.model.PatchRequest; +import com.crowdin.client.labels.model.Label; +import com.crowdin.client.projectsgroups.model.Type; +import com.crowdin.client.sourcefiles.model.Branch; +import com.crowdin.client.sourcefiles.model.FileInfo; +import com.crowdin.client.sourcestrings.model.SourceString; +import lombok.AllArgsConstructor; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.*; +import java.util.stream.Collectors; + +import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE; +import static com.crowdin.cli.utils.console.ExecutionStatus.OK; +import static com.crowdin.cli.utils.console.ExecutionStatus.WARNING; +import static java.util.Objects.nonNull; + +@AllArgsConstructor +class ContextResetAction implements NewAction { + + private final List filesFilter; + private final List labelsFilter; + private final String branchFilter; + private final String croqlFilter; + private final LocalDate sinceFilter; + private final boolean dryRun; + private final int batchSize; + private final boolean plainView; + private final boolean noProgress; + + @Override + public void act(Outputter out, ProjectProperties pb, ProjectClient client) { + + CrowdinProjectFull project = ConsoleSpinner.execute( + out, + "message.spinner.fetching_project_info", + "error.collect_project_info", + this.noProgress, + this.plainView, + () -> client.downloadFullProject(this.branchFilter) + ); + + boolean isStringsBasedProject = Objects.equals(project.getType(), Type.STRINGS_BASED); + + Long branchId = Optional.ofNullable(project.getBranch()) + .map(Branch::getId) + .orElse(null); + + Map labelsMap = client.listLabels().stream() + .collect(Collectors.toMap(Label::getTitle, Label::getId, (existing, replacement) -> existing)); + + String filterLabelsStr = Optional + .ofNullable(this.labelsFilter) + .filter(list -> !list.isEmpty()) + .map(list -> list.stream() + .filter(labelsMap::containsKey) + .map(label -> labelsMap.get(label).toString()) + .collect(Collectors.joining(","))) + .orElse(null); + + String encodedCroql = nonNull(croqlFilter) ? Utils.encodeURL(croqlFilter) : null; + + List fileIds = null; + + if (!isStringsBasedProject && nonNull(this.filesFilter) && !this.filesFilter.isEmpty()) { + fileIds = project.getFileInfos().stream() + .filter(file -> this.filesFilter.stream().anyMatch(filter -> GlobUtil.matches(filter, file.getPath()))) + .map(FileInfo::getId) + .toList(); + } + + List strings = new ArrayList<>(); + + if (isStringsBasedProject) { + strings = client.listSourceString(null, branchId, filterLabelsStr, null, encodedCroql, null, null); + } else { + if (nonNull(fileIds) && !fileIds.isEmpty()) { + for (Long fileId : fileIds) { + strings.addAll(client.listSourceString(fileId, branchId, filterLabelsStr, null, encodedCroql, null, null)); + } + } else { + strings = client.listSourceString(null, branchId, filterLabelsStr, null, encodedCroql, null, null); + } + } + + strings = strings.stream() + .filter(string -> { + if (sinceFilter == null) { + return true; + } + var createdDate = string.getCreatedAt().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + + return createdDate.isAfter(sinceFilter) || createdDate.isEqual(sinceFilter); + }) + .toList(); + + strings = strings.stream() + .filter(string -> !AiContextUtil.getAiContextSection(string.getContext()).isEmpty()) + .toList(); + + if (strings.isEmpty()) { + out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("message.source_string_list_not_found"))); + return; + } + + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.downloaded_strings"), strings.size()))); + + if (dryRun) { + strings.forEach(string -> { + if (!plainView) { + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.string_reset_dryrun"), string.getId(), string.getText(), AiContextUtil.getManualContext(string.getContext())))); + } else { + out.println(string.getId() + ": " + string.getText() + " | " + AiContextUtil.getManualContext(string.getContext())); + } + }); + return; + } + + List batchContextEdit = strings.stream() + .map(string -> { + var request = new PatchRequest(); + request.setOp(PatchOperation.REPLACE); + request.setPath("/" + string.getId() + "/context"); + request.setValue(AiContextUtil.getManualContext(string.getContext())); + return request; + }) + .toList(); + + for (int i = 0; i < batchContextEdit.size(); i += batchSize) { + int end = Math.min(i + batchSize, batchContextEdit.size()); + client.batchEditSourceStrings(batchContextEdit.subList(i, end)); + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.strings_reset_success"), end, batchContextEdit.size()))); + } + } +} diff --git a/src/main/java/com/crowdin/cli/commands/actions/ContextStatusAction.java b/src/main/java/com/crowdin/cli/commands/actions/ContextStatusAction.java new file mode 100644 index 000000000..82ac9fb3d --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/actions/ContextStatusAction.java @@ -0,0 +1,196 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.CrowdinProjectFull; +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.cli.utils.AiContextUtil; +import com.crowdin.cli.utils.GlobUtil; +import com.crowdin.cli.utils.Utils; +import com.crowdin.cli.utils.console.ConsoleSpinner; +import com.crowdin.client.labels.model.Label; +import com.crowdin.client.projectsgroups.model.Type; +import com.crowdin.client.sourcefiles.model.Branch; +import com.crowdin.client.sourcefiles.model.FileInfo; +import com.crowdin.client.sourcestrings.model.SourceString; +import lombok.AllArgsConstructor; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Objects.nonNull; + +@AllArgsConstructor +class ContextStatusAction implements NewAction { + + private final List filesFilter; + private final List labelsFilter; + private final String branchFilter; + private final String croqlFilter; + private final LocalDate sinceFilter; + private final boolean byFile; + private final boolean plainView; + private final boolean noProgress; + + @Override + public void act(Outputter out, ProjectProperties pb, ProjectClient client) { + + CrowdinProjectFull project = ConsoleSpinner.execute( + out, + "message.spinner.fetching_project_info", + "error.collect_project_info", + this.noProgress, + this.plainView, + () -> client.downloadFullProject(this.branchFilter) + ); + + boolean isStringsBasedProject = Objects.equals(project.getType(), Type.STRINGS_BASED); + + Long branchId = Optional.ofNullable(project.getBranch()) + .map(Branch::getId) + .orElse(null); + + Map labelsMap = client.listLabels().stream() + .collect(Collectors.toMap(Label::getTitle, Label::getId, (existing, replacement) -> existing)); + + String filterLabelsStr = Optional + .ofNullable(this.labelsFilter) + .filter(list -> !list.isEmpty()) + .map(list -> list.stream() + .filter(labelsMap::containsKey) + .map(label -> labelsMap.get(label).toString()) + .collect(Collectors.joining(","))) + .orElse(null); + + String encodedCroql = nonNull(croqlFilter) ? Utils.encodeURL(croqlFilter) : null; + + List fileIds = null; + + if (!isStringsBasedProject && nonNull(this.filesFilter) && !this.filesFilter.isEmpty()) { + fileIds = project.getFileInfos().stream() + .filter(file -> this.filesFilter.stream().anyMatch(filter -> GlobUtil.matches(filter, file.getPath()))) + .map(FileInfo::getId) + .toList(); + } + + List strings = new ArrayList<>(); + + if (isStringsBasedProject) { + strings = client.listSourceString(null, branchId, filterLabelsStr, null, encodedCroql, null, null); + } else { + if (nonNull(fileIds) && !fileIds.isEmpty()) { + for (Long fileId : fileIds) { + strings.addAll(client.listSourceString(fileId, branchId, filterLabelsStr, null, encodedCroql, null, null)); + } + } else { + strings = client.listSourceString(null, branchId, filterLabelsStr, null, encodedCroql, null, null); + } + } + + strings = strings.stream() + .filter(string -> { + if (sinceFilter == null) { + return true; + } + var createdDate = string.getCreatedAt().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + + return createdDate.isAfter(sinceFilter) || createdDate.isEqual(sinceFilter); + }) + .toList(); + + out.println(String.format("Context Status for Project \"%s\" (ID: %d)", project.getProject().getName(), project.getProjectId())); + out.println(""); + + if (byFile && !isStringsBasedProject) { + Map filesMap = project.getFileInfos().stream() + .collect(Collectors.toMap(FileInfo::getId, FileInfo::getPath)); + Map groupByFile = strings + .stream() + .collect( + Collectors.groupingBy( + string -> filesMap.get(string.getFileId()), + Collectors.collectingAndThen( + Collectors.toList(), + this::calculateStats + ) + ) + ); + int longestFilePathLength = groupByFile.keySet().stream().mapToInt(String::length).max().orElse(0); + + String headerFormat = "%-" + longestFilePathLength + "s %8s %14s %8s"; + String rowFormat = "%-" + longestFilePathLength + "s %8d %6d (%.2f%%) %8d"; + + out.println(String.format( + headerFormat, + "File", + "Total", + "AI Context", + "Missing" + )); + + groupByFile.forEach((path, stats) -> { + out.println(String.format( + rowFormat, + path, + stats.total, + stats.withAi, + stats.withAiPercentage, + stats.total - stats.withAi + )); + }); + + } else { + var stats = calculateStats(strings); + + out.println(String.format("Total strings: %d", stats.total)); + out.println(String.format("With AI context: %d (%.2f%%)", stats.withAi, stats.withAiPercentage)); + out.println(String.format("Without AI context: %d (%.2f%%)", stats.withoutAi, stats.withoutAiPercentage)); + out.println(String.format("With manual context: %d (%.2f%%)", stats.withManual, stats.withManualPercentage)); + } + + out.println(""); + out.println("Run 'crowdin context download --status=empty' to export strings needing context."); + } + + private Stats calculateStats(List strings) { + var stringsWithAiContext = strings.stream() + .filter(string -> !AiContextUtil.getAiContextSection(string.getContext()).isEmpty()) + .count(); + + var stringsWithoutAiContext = strings.stream() + .filter(string -> AiContextUtil.getAiContextSection(string.getContext()).isEmpty()) + .count(); + + var stringsWithManualContext = strings.stream() + .filter(string -> !AiContextUtil.getManualContext(string.getContext()).isEmpty()) + .count(); + + return new Stats( + strings.size(), + stringsWithAiContext, + percentageValue(stringsWithAiContext, strings.size()), + stringsWithoutAiContext, + percentageValue(stringsWithoutAiContext, strings.size()), + stringsWithManualContext, + percentageValue(stringsWithManualContext, strings.size()) + ); + } + + private record Stats( + long total, + long withAi, + double withAiPercentage, + long withoutAi, + double withoutAiPercentage, + long withManual, + double withManualPercentage + ) { + } + + private double percentageValue(long target, long total) { + return (double) target / total * 100; + } +} diff --git a/src/main/java/com/crowdin/cli/commands/actions/ContextUploadAction.java b/src/main/java/com/crowdin/cli/commands/actions/ContextUploadAction.java new file mode 100644 index 000000000..ded68aa00 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/actions/ContextUploadAction.java @@ -0,0 +1,71 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.cli.utils.AiContextUtil; +import com.crowdin.client.core.model.PatchOperation; +import com.crowdin.client.core.model.PatchRequest; +import lombok.AllArgsConstructor; + +import java.io.File; +import java.util.AbstractMap; +import java.util.List; + +import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE; +import static com.crowdin.cli.utils.console.ExecutionStatus.OK; + +@AllArgsConstructor +class ContextUploadAction implements NewAction { + + private final File file; + private final boolean overwrite; + private final boolean dryRun; + private final boolean plainView; + private final int batchSize; + + @Override + public void act(Outputter out, ProjectProperties pb, ProjectClient client) { + var stringContextRecords = AiContextUtil.readRecords(file); + + if (overwrite) { + stringContextRecords = stringContextRecords.stream() + .filter(record -> record.getAi_context() == null || record.getAi_context().isEmpty()) + .toList(); + } + + var recordsWithContext = stringContextRecords + .stream() + .map(record -> new AbstractMap.SimpleEntry<>(record, AiContextUtil.fullContext(record.getContext(), record.getAi_context()))) + .toList(); + + if (dryRun) { + recordsWithContext.forEach(record -> { + if (!plainView) { + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.string_upload_dryrun"), record.getKey().getId(), record.getKey().getText(), record.getValue()))); + } else { + out.println(record.getKey().getId() + ": " + record.getKey().getText() + " | " + record.getValue()); + } + }); + return; + } + + List batchContextEdit = recordsWithContext.stream() + .map(record -> { + var request = new PatchRequest(); + request.setOp(PatchOperation.REPLACE); + request.setPath("/" + record.getKey().getId() + "/context"); + request.setValue(record.getValue()); + return request; + }) + .toList(); + + for (int i = 0; i < batchContextEdit.size(); i += batchSize) { + int end = Math.min(i + batchSize, batchContextEdit.size()); + client.batchEditSourceStrings(batchContextEdit.subList(i, end)); + out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("messages.context.strings_upload_success"), end, batchContextEdit.size()))); + } + + } +} diff --git a/src/main/java/com/crowdin/cli/commands/actions/StringListAction.java b/src/main/java/com/crowdin/cli/commands/actions/StringListAction.java index 70841b53a..312bb388d 100644 --- a/src/main/java/com/crowdin/cli/commands/actions/StringListAction.java +++ b/src/main/java/com/crowdin/cli/commands/actions/StringListAction.java @@ -8,6 +8,7 @@ import com.crowdin.cli.commands.functionality.ProjectFilesUtils; import com.crowdin.cli.commands.picocli.ExitCodeExceptionMapper; import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.cli.utils.StringUtil; import com.crowdin.cli.utils.Utils; import com.crowdin.cli.utils.console.ConsoleSpinner; import com.crowdin.client.labels.model.Label; @@ -111,18 +112,7 @@ public static void printSourceString( String labelsString = (ss.getLabelIds() != null) ? ss.getLabelIds().stream().map(labelsMap::get).map(s -> String.format("@|cyan %s|@", s)).collect(Collectors.joining(", ")) : ""; - StringBuilder text = new StringBuilder(); - if (ss.getText() instanceof HashMap) { - HashMap map = (HashMap) ss.getText(); - for (Map.Entry entry : map.entrySet()) { - text.append(entry.getKey()).append(": ").append(entry.getValue()).append(" | "); - } - if (text.length() > 0) { - text.delete(text.length() - 3, text.length()); - } - } else { - text.append((String) ss.getText()); - } + String text = StringUtil.getStringText(ss); if (!plainView) { if (ss.getIdentifier() == null) { out.println(String.format(RESOURCE_BUNDLE.getString("message.source_string_list_text_short"), ss.getId(), text)); diff --git a/src/main/java/com/crowdin/cli/commands/picocli/CommandNames.java b/src/main/java/com/crowdin/cli/commands/picocli/CommandNames.java index 0e5f77764..2dbaae79f 100644 --- a/src/main/java/com/crowdin/cli/commands/picocli/CommandNames.java +++ b/src/main/java/com/crowdin/cli/commands/picocli/CommandNames.java @@ -68,4 +68,10 @@ public final class CommandNames { public static final String APP = "app"; public static final String UNINSTALL = "uninstall"; public static final String INSTALL = "install"; + + public static final String CONTEXT = "context"; + public static final String CONTEXT_DOWNLOAD = "download"; + public static final String CONTEXT_UPLOAD = "upload"; + public static final String CONTEXT_RESET = "reset"; + public static final String CONTEXT_STATUS = "status"; } diff --git a/src/main/java/com/crowdin/cli/commands/picocli/ContextDownloadSubcommand.java b/src/main/java/com/crowdin/cli/commands/picocli/ContextDownloadSubcommand.java new file mode 100644 index 000000000..d4d222bd6 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/picocli/ContextDownloadSubcommand.java @@ -0,0 +1,85 @@ +package com.crowdin.cli.commands.picocli; + +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.Actions; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.commands.functionality.FsFiles; +import com.crowdin.cli.properties.ProjectProperties; +import picocli.CommandLine; + +import java.io.File; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@CommandLine.Command( + name = CommandNames.CONTEXT_DOWNLOAD, + sortOptions = false +) +class ContextDownloadSubcommand extends ActCommandProject { + + private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final Set AVAILABLE_FORMATS = Set.of("jsonl"); + private static final Set AVAILABLE_STATUSES = Set.of("ai", "empty", "manual"); + + @CommandLine.Parameters(descriptionKey = "crowdin.context.download.to") + protected File to; + + @CommandLine.Option(names = {"-f", "--file"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.download.file") + protected List files; + + @CommandLine.Option(names = {"--label"}, paramLabel = "...", descriptionKey = "crowdin.context.download.label", order = -2) + protected List labelNames; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.download.branch") + protected String branchName; + + @CommandLine.Option(names = {"--croql"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.download.croql") + protected String croql; + + @CommandLine.Option(names = {"--since"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.download.since") + protected String since; + + @CommandLine.Option(names = {"--format"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.download.format") + protected String format = "jsonl"; + + @CommandLine.Option(names = {"--status"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.download.status") + protected String status; + + @CommandLine.Option(names = {"--plain"}, descriptionKey = "crowdin.list.usage.plain") + protected boolean plainView; + + @Override + protected List checkOptions() { + List errors = new ArrayList<>(); + if (since != null) { + try { + LocalDate.parse(since, DATE_FORMAT); + } catch (Exception e) { + errors.add(RESOURCE_BUNDLE.getString("error.context.invalid_since")); + } + } + + if (format != null) { + if (!AVAILABLE_FORMATS.contains(format)) { + errors.add(RESOURCE_BUNDLE.getString("error.context.invalid_format")); + } + } + + if (status != null) { + if (!AVAILABLE_STATUSES.contains(status)) { + errors.add(RESOURCE_BUNDLE.getString("error.context.invalid_status")); + } + } + + return errors; + } + + @Override + protected NewAction getAction(Actions actions) { + var sinceDate = since != null ? LocalDate.parse(since, DATE_FORMAT) : null; + return actions.contextDownload(to, files, labelNames, branchName, croql, sinceDate, status, format, new FsFiles(), plainView, noProgress); + } +} diff --git a/src/main/java/com/crowdin/cli/commands/picocli/ContextResetSubcommand.java b/src/main/java/com/crowdin/cli/commands/picocli/ContextResetSubcommand.java new file mode 100644 index 000000000..8ee46fa55 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/picocli/ContextResetSubcommand.java @@ -0,0 +1,79 @@ +package com.crowdin.cli.commands.picocli; + +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.Actions; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.properties.ProjectProperties; +import picocli.CommandLine; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command( + name = CommandNames.CONTEXT_RESET, + sortOptions = false +) +class ContextResetSubcommand extends ActCommandProject { + + private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + @CommandLine.Option(names = {"-f", "--file"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.file") + protected List files; + + @CommandLine.Option(names = {"--label"}, paramLabel = "...", descriptionKey = "crowdin.context.reset.label", order = -2) + protected List labelNames; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.branch") + protected String branchName; + + @CommandLine.Option(names = {"--croql"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.croql") + protected String croql; + + @CommandLine.Option(names = {"--since"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.since") + protected String since; + + @CommandLine.Option(names = {"--dry-run"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.dryrun") + protected boolean dryrun; + + @CommandLine.Option(names = {"--all"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.all") + protected boolean all; + + @CommandLine.Option(names = {"--batch-size"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.reset.batchSize") + protected int batchSize = 100; + + @CommandLine.Option(names = {"--plain"}, descriptionKey = "crowdin.list.usage.plain") + protected boolean plainView; + + @Override + protected List checkOptions() { + List errors = new ArrayList<>(); + + if (since != null) { + try { + LocalDate.parse(since, DATE_FORMAT); + } catch (Exception e) { + errors.add(RESOURCE_BUNDLE.getString("error.context.invalid_since")); + } + } + + var isAllEmpty = (files == null || files.isEmpty()) + && (labelNames == null || labelNames.isEmpty()) + && (branchName == null || branchName.isEmpty()) + && (croql == null || croql.isEmpty()) + && (since == null); + + if (isAllEmpty && !all) { + errors.add(RESOURCE_BUNDLE.getString("error.context.no_all_flag")); + } + + return errors; + } + + @Override + protected NewAction getAction(Actions actions) { + var sinceDate = since != null ? LocalDate.parse(since, DATE_FORMAT) : null; + return actions.contextReset(files, labelNames, branchName, croql, sinceDate, dryrun, batchSize, plainView, noProgress); + } +} diff --git a/src/main/java/com/crowdin/cli/commands/picocli/ContextStatusSubcommand.java b/src/main/java/com/crowdin/cli/commands/picocli/ContextStatusSubcommand.java new file mode 100644 index 000000000..37f8c516b --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/picocli/ContextStatusSubcommand.java @@ -0,0 +1,63 @@ +package com.crowdin.cli.commands.picocli; + +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.Actions; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.properties.ProjectProperties; +import picocli.CommandLine; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command( + name = CommandNames.CONTEXT_STATUS, + sortOptions = false +) +class ContextStatusSubcommand extends ActCommandProject { + + private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + @CommandLine.Option(names = {"-f", "--file"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.status.file") + protected List files; + + @CommandLine.Option(names = {"--label"}, paramLabel = "...", descriptionKey = "crowdin.context.status.label", order = -2) + protected List labelNames; + + @CommandLine.Option(names = {"-b", "--branch"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.status.branch") + protected String branchName; + + @CommandLine.Option(names = {"--croql"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.status.croql") + protected String croql; + + @CommandLine.Option(names = {"--since"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.status.since") + protected String since; + + @CommandLine.Option(names = {"--by-file"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.status.byFile") + protected boolean byFile; + + @CommandLine.Option(names = {"--plain"}, descriptionKey = "crowdin.list.usage.plain") + protected boolean plainView; + + @Override + protected List checkOptions() { + List errors = new ArrayList<>(); + + if (since != null) { + try { + LocalDate.parse(since, DATE_FORMAT); + } catch (Exception e) { + errors.add(RESOURCE_BUNDLE.getString("error.context.invalid_since")); + } + } + + return errors; + } + + @Override + protected NewAction getAction(Actions actions) { + var sinceDate = since != null ? LocalDate.parse(since, DATE_FORMAT) : null; + return actions.contextStatus(files, labelNames, branchName, croql, sinceDate, byFile, plainView, noProgress); + } +} diff --git a/src/main/java/com/crowdin/cli/commands/picocli/ContextSubcommand.java b/src/main/java/com/crowdin/cli/commands/picocli/ContextSubcommand.java new file mode 100644 index 000000000..8c8ad1d68 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/picocli/ContextSubcommand.java @@ -0,0 +1,19 @@ +package com.crowdin.cli.commands.picocli; + +import picocli.CommandLine; + +@CommandLine.Command( + name = CommandNames.CONTEXT, + subcommands = { + ContextDownloadSubcommand.class, + ContextUploadSubcommand.class, + ContextResetSubcommand.class, + ContextStatusSubcommand.class, + } +) +class ContextSubcommand extends HelpCommand { + @Override + protected CommandLine getCommand(CommandLine rootCommand) { + return rootCommand.getSubcommands().get(CommandNames.CONTEXT); + } +} diff --git a/src/main/java/com/crowdin/cli/commands/picocli/ContextUploadSubcommand.java b/src/main/java/com/crowdin/cli/commands/picocli/ContextUploadSubcommand.java new file mode 100644 index 000000000..2caca6df2 --- /dev/null +++ b/src/main/java/com/crowdin/cli/commands/picocli/ContextUploadSubcommand.java @@ -0,0 +1,49 @@ +package com.crowdin.cli.commands.picocli; + +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.Actions; +import com.crowdin.cli.commands.NewAction; +import com.crowdin.cli.properties.ProjectProperties; +import picocli.CommandLine; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command( + name = CommandNames.CONTEXT_UPLOAD, + sortOptions = false +) +class ContextUploadSubcommand extends ActCommandProject { + + @CommandLine.Parameters(descriptionKey = "crowdin.context.upload.file") + protected File file; + + @CommandLine.Option(names = {"--overwrite"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.upload.overwrite") + protected boolean overwrite; + + @CommandLine.Option(names = {"--dry-run"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.upload.dryrun") + protected boolean dryrun; + + @CommandLine.Option(names = {"--batch-size"}, paramLabel = "...", order = -2, descriptionKey = "crowdin.context.upload.batchSize") + protected int batchSize = 100; + + @CommandLine.Option(names = {"--plain"}, descriptionKey = "crowdin.list.usage.plain") + protected boolean plainView; + + @Override + protected List checkOptions() { + List errors = new ArrayList<>(); + + if (!file.exists()) { + errors.add(String.format(RESOURCE_BUNDLE.getString("error.file_not_found"), file)); + } + + return errors; + } + + @Override + protected NewAction getAction(Actions actions) { + return actions.contextUpload(file, overwrite, this.dryrun, plainView, batchSize); + } +} diff --git a/src/main/java/com/crowdin/cli/commands/picocli/RootCommand.java b/src/main/java/com/crowdin/cli/commands/picocli/RootCommand.java index 1624886ca..ac6427558 100644 --- a/src/main/java/com/crowdin/cli/commands/picocli/RootCommand.java +++ b/src/main/java/com/crowdin/cli/commands/picocli/RootCommand.java @@ -26,7 +26,8 @@ ConfigSubcommand.class, ProjectSubcommand.class, CompletionSubCommand.class, - ApplicationSubcommand.class + ApplicationSubcommand.class, + ContextSubcommand.class, }) class RootCommand extends HelpCommand { @Override diff --git a/src/main/java/com/crowdin/cli/utils/AiContextUtil.java b/src/main/java/com/crowdin/cli/utils/AiContextUtil.java new file mode 100644 index 000000000..e897cd726 --- /dev/null +++ b/src/main/java/com/crowdin/cli/utils/AiContextUtil.java @@ -0,0 +1,88 @@ +package com.crowdin.cli.utils; + +import lombok.Data; +import lombok.SneakyThrows; +import org.json.JSONObject; + +import java.io.File; +import java.nio.file.Files; +import java.util.List; +import java.util.Objects; + +public class AiContextUtil { + + private static final String AI_CONTEXT_SECTION_START = "\n\n✨ AI Context\n"; + private static final String AI_CONTEXT_SECTION_END = "\n✨ 🔚"; + + private AiContextUtil() { + } + + public static String getManualContext(String context) { + if (context == null || context.isEmpty()) { + return ""; + } + + int startIndex = context.indexOf(AI_CONTEXT_SECTION_START); + if (startIndex != -1) { + return context.substring(0, startIndex).trim(); + } + + return context.trim(); + } + + public static String getAiContextSection(String context) { + if (context == null || context.isEmpty()) { + return ""; + } + + int startIndex = context.indexOf(AI_CONTEXT_SECTION_START); + int endIndex = context.indexOf(AI_CONTEXT_SECTION_END); + + if (startIndex != -1 && endIndex != -1 && startIndex < endIndex) { + return context.substring(startIndex + AI_CONTEXT_SECTION_START.length(), endIndex); + } + + return ""; + } + + public static String fullContext(String manualContext, String aiContext) { + StringBuilder fullContext = new StringBuilder(manualContext.trim()); + if (aiContext != null && !aiContext.isEmpty()) { + fullContext.append(AI_CONTEXT_SECTION_START).append(aiContext.trim()).append(AI_CONTEXT_SECTION_END); + } + return fullContext.toString(); + } + + @SneakyThrows + public static List readRecords(File file) { + return Files.readAllLines(file.toPath()) + .stream() + .map(line -> { + try { + var object = new JSONObject(line); + return new StringContextRecord( + object.getLong("id"), + object.getString("key"), + object.getString("text"), + object.getString("file"), + object.getString("context"), + object.getString("ai_context") + ); + } catch (Exception e) { + return null; + } + }) + .filter(Objects::nonNull) + .toList(); + } + + @Data + public static class StringContextRecord { + private final Long id; + private final String key; + private final String text; + private final String file; + private final String context; + private final String ai_context; + } +} diff --git a/src/main/java/com/crowdin/cli/utils/GlobUtil.java b/src/main/java/com/crowdin/cli/utils/GlobUtil.java new file mode 100644 index 000000000..a892958c3 --- /dev/null +++ b/src/main/java/com/crowdin/cli/utils/GlobUtil.java @@ -0,0 +1,85 @@ +package com.crowdin.cli.utils; + +import java.util.regex.Pattern; + +public class GlobUtil { + + private GlobUtil() { + } + + /** + * Matches text against a glob expression. + */ + public static boolean matches(String glob, String text) { + String regex = toRegex(glob); + return Pattern.matches(regex, text); + } + + /** + * Converts glob pattern to Java regex. + */ + public static String toRegex(String glob) { + StringBuilder regex = new StringBuilder(); + int length = glob.length(); + boolean inGroup = false; + + for (int i = 0; i < length; i++) { + char c = glob.charAt(i); + + switch (c) { + case '*': + if (i + 1 < length && glob.charAt(i + 1) == '*') { + // ** -> any directories + regex.append(".*"); + i++; + } else { + // * -> any except separator + regex.append("[^/]*"); + } + break; + + case '?': + regex.append('.'); + break; + + case '.': + case '(': + case ')': + case '+': + case '|': + case '^': + case '$': + case '@': + case '%': + regex.append("\\").append(c); + break; + + case '\\': + if (i + 1 < length) { + regex.append("\\").append(glob.charAt(++i)); + } + break; + + case '[': + inGroup = true; + regex.append('['); + break; + + case ']': + inGroup = false; + regex.append(']'); + break; + + case '!': + if (inGroup) regex.append('^'); + else regex.append('!'); + break; + + default: + regex.append(c); + } + } + + return "^" + regex + "$"; + } +} diff --git a/src/main/java/com/crowdin/cli/utils/StringUtil.java b/src/main/java/com/crowdin/cli/utils/StringUtil.java new file mode 100644 index 000000000..7e655c395 --- /dev/null +++ b/src/main/java/com/crowdin/cli/utils/StringUtil.java @@ -0,0 +1,25 @@ +package com.crowdin.cli.utils; + +import com.crowdin.client.sourcestrings.model.SourceString; + +import java.util.HashMap; +import java.util.Map; + +public class StringUtil { + + public static String getStringText(SourceString ss) { + StringBuilder text = new StringBuilder(); + if (ss.getText() instanceof HashMap) { + HashMap map = (HashMap) ss.getText(); + for (Map.Entry entry : map.entrySet()) { + text.append(entry.getKey()).append(": ").append(entry.getValue()).append(" | "); + } + if (!text.isEmpty()) { + text.delete(text.length() - 3, text.length()); + } + } else { + text.append((String) ss.getText()); + } + return text.toString(); + } +} diff --git a/src/main/resources/messages/messages.properties b/src/main/resources/messages/messages.properties index c4d339f95..ffb70c908 100755 --- a/src/main/resources/messages/messages.properties +++ b/src/main/resources/messages/messages.properties @@ -522,6 +522,52 @@ crowdin.app.uninstall.usage.customSynopsis=@|fg(green) crowdin app uninstall|@ < crowdin.app.uninstall.identifier=Application identifier crowdin.app.uninstall.force=Force to delete application installation +# CROWDIN CONTEXT COMMAND +crowdin.context.usage.description=Manage strings context +crowdin.context.usage.customSynopsis=@|fg(green) crowdin context|@ [SUBCOMMAND] [CONFIG OPTIONS] [OPTIONS] + +# CROWDIN CONTEXT DOWNLOAD +crowdin.context.download.usage.description=Download strings context +crowdin.context.download.usage.customSynopsis=@|fg(green) crowdin context download|@ [CONFIG OPTIONS] [OPTIONS] +crowdin.context.download.to=File path to download the context to +crowdin.context.download.file=Filter strings by Crowdin file path (glob). May be specified multiple times +crowdin.context.download.label=Filter strings by label. May be specified multiple times +crowdin.context.download.branch=Filter by branch name +crowdin.context.download.croql=CroQL expression +crowdin.context.download.since=Only strings created after this date (YYYY-MM-DD) +crowdin.context.download.format=Output format (only option: jsonl) +crowdin.context.download.status=Filter by context status (empty, ai, manual) + +# CROWDIN CONTEXT UPLOAD +crowdin.context.upload.usage.description=Upload strings context +crowdin.context.upload.usage.customSynopsis=@|fg(green) crowdin context upload|@ [CONFIG OPTIONS] [OPTIONS] +crowdin.context.upload.file=File path to upload context from +crowdin.context.upload.overwrite=Also update strings where ai_context is empty (removes their AI section). Default: false +crowdin.context.upload.dryrun=Show what would be updated without making changes. Default: false +crowdin.context.upload.batchSize=Number of strings per API batch request. Default: 100 + +# CROWDIN CONTEXT RESET +crowdin.context.reset.usage.description=Remove AI-generated context from strings, preserving manual context +crowdin.context.reset.usage.customSynopsis=@|fg(green) crowdin context reset|@ [CONFIG OPTIONS] [OPTIONS] +crowdin.context.reset.file=Only reset strings from matching files. May be specified multiple times +crowdin.context.reset.label=Only reset strings from matching labels. May be specified multiple times +crowdin.context.reset.branch=Only reset strings from matching branch +crowdin.context.reset.croql=Only reset strings from matching CroQL expression +crowdin.context.reset.since=Only reset strings that were created after this date (YYYY-MM-DD) +crowdin.context.reset.dryrun=Show what would be updated without making changes. Default: false +crowdin.context.reset.batchSize=Number of strings per API batch request. Default: 100 +crowdin.context.reset.all=Required safety flag when no filter is specified (n/a) + +# CROWDIN CONTEXT STATUS +crowdin.context.status.usage.description=Show context coverage statistics +crowdin.context.status.usage.customSynopsis=@|fg(green) crowdin context status|@ [CONFIG OPTIONS] [OPTIONS] +crowdin.context.status.file=Filter strings by Crowdin file path (glob). May be specified multiple times +crowdin.context.status.label=Filter strings by label. May be specified multiple times +crowdin.context.status.branch=Filter by branch name +crowdin.context.status.croql=CroQL expression +crowdin.context.status.since=Only strings created after this date (YYYY-MM-DD) +crowdin.context.status.byFile=Break down stats per file + error.collect_project_info=Failed to collect project info. Please contact our support team for help error.no_sources=No sources found for '%s' pattern. Check the source paths in your configuration file error.only_enterprise=Operation is available only for Crowdin Enterprise @@ -715,6 +761,11 @@ error.file_not_found=File '%s' not found in the Crowdin project error.local_file_not_found=File '%s' not found by the specified path error.branch_required_string_project=Branch is required for string-based projects +error.context.invalid_since=The '--since' parameter should be in 'YYYY-MM-DD' format +error.context.invalid_status=The '--status' parameter has an invalid value. Supported values: empty, ai, manual +error.context.invalid_format=The '--format' parameter has an invalid value. Supported value: jsonl +error.context.no_all_flag=The '--all' parameter should be specified explicitly if no other filter was provided + message.config.read=Loading configuration from the '%s' file message.creds.read=Loading credentials from the '%s' file message.new_version_text=New version of Crowdin CLI is available! %s -> %s @@ -859,6 +910,13 @@ message.no_manager_access_in_upload_sources=You must have @|yellow manager or de message.no_manager_access_in_upload_sources_dryrun=You must have @|yellow manager or developer role|@ in the project to apply 'delete-obsolete' options message.file_context_for_string_based_project=Context can not be used for string-based projects +messages.context.downloaded_strings=Downloaded %d strings +messages.context.saved_strings=@|green '%s' saved successfully|@ +messages.context.string_upload_dryrun=String #%d: %s (context: %s) would be uploaded +messages.context.strings_upload_success=Updated strings %d/%d +messages.context.string_reset_dryrun=String #%d: %s (context: %s) would be updated +messages.context.strings_reset_success=Updated strings %d/%d + message.warning.not_yml=File '%s' is not a YAML or YML file message.warning.browser_not_found=\nError opening a web browser. Please open the following link manually:\n%s message.warning.auto_approve_option_with_mt='--auto-approve-option' is used only for the TM Pre-Translation method diff --git a/src/test/java/com/crowdin/cli/client/ProjectBuilder.java b/src/test/java/com/crowdin/cli/client/ProjectBuilder.java index d2d645b06..91ebbe1bb 100644 --- a/src/test/java/com/crowdin/cli/client/ProjectBuilder.java +++ b/src/test/java/com/crowdin/cli/client/ProjectBuilder.java @@ -45,6 +45,7 @@ public static ProjectBuilder emptyProject(Long projectId) { project.setBranches(branches); project.setProjectLanguages(projectLanguages); project.setAccessLevel(CrowdinProject.Access.MANAGER); + project.setProject(new ProjectSettings()); projectBuilder.project = project; projectBuilder.projectSettings = projectSettings; return projectBuilder; diff --git a/src/test/java/com/crowdin/cli/commands/actions/ContextDownloadActionTest.java b/src/test/java/com/crowdin/cli/commands/actions/ContextDownloadActionTest.java new file mode 100644 index 000000000..2364a52f1 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/actions/ContextDownloadActionTest.java @@ -0,0 +1,192 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.ProjectBuilder; +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.client.models.SourceStringBuilder; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.commands.functionality.FilesInterface; +import com.crowdin.cli.properties.NewProjectPropertiesUtilBuilder; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.client.projectsgroups.model.Type; +import com.crowdin.client.sourcestrings.model.SourceString; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.io.InputStream; +import java.io.File; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +public class ContextDownloadActionTest { + + @Test + public void testJsonlSavesSingleString() throws Exception { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + + // Build project with one file (id=101) + var projectFull = ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId())) + .addFile("first.txt", "plain", 101L, null, null, "/%original_file_name%") + .build(); + projectFull.setType(Type.FILES_BASED); + + ProjectClient client = mock(ProjectClient.class); + when(client.downloadFullProject(null)).thenReturn(projectFull); + when(client.listLabels()).thenReturn(List.of()); + + SourceString ss = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(701L, "the-text", "manual\n\n✨ AI Context\nai-content\n✨ 🔚", "the.key", 101L) + .build(); + + when(client.listSourceString(null, null, null, null, null, null, null)) + .thenReturn(Arrays.asList(ss)); + + FilesInterface files = mock(FilesInterface.class); + File to = new File("out.jsonl"); + + ContextDownloadAction action = new ContextDownloadAction( + to, + null, + null, + null, + null, + null, + null, + "jsonl", + files, + true, + false + ); + + action.act(Outputter.getDefault(), pb, client); + + verify(client).downloadFullProject(null); + verify(client).listLabels(); + verify(client).listSourceString(null, null, null, null, null, null, null); + + ArgumentCaptor captor = ArgumentCaptor.forClass(InputStream.class); + verify(files).writeToFile(eq(to.toString()), captor.capture()); + + try (InputStream is = captor.getValue()) { + byte[] bytes = is.readAllBytes(); + String content = new String(bytes, UTF_8); + // The saved jsonl should contain id and key + assertTrue(content.contains("\"id\":701")); + assertTrue(content.contains("\"key\":\"the.key\"")); + assertTrue(content.contains("\"ai_context\":\"ai-content\"")); + } + + verifyNoMoreInteractions(files); + } + + @Test + public void testStatusFilterAi() throws Exception { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + + var projectFull = ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId())) + .addFile("first.txt", "plain", 101L, null, null, "/%original_file_name%") + .build(); + projectFull.setType(Type.FILES_BASED); + + ProjectClient client = mock(ProjectClient.class); + when(client.downloadFullProject(null)).thenReturn(projectFull); + when(client.listLabels()).thenReturn(List.of()); + + SourceString emptyCtx = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(1L, "t1", "", "k1", 101L).build(); + SourceString aiCtx = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(2L, "t2", "manual\n\n✨ AI Context\naiOnly\n✨ 🔚", "k2", 101L).build(); + SourceString manualCtx = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(3L, "t3", "only manual", "k3", 101L).build(); + + when(client.listSourceString(null, null, null, null, null, null, null)) + .thenReturn(Arrays.asList(emptyCtx, aiCtx, manualCtx)); + + FilesInterface files = mock(FilesInterface.class); + File to = new File("out2.jsonl"); + + ContextDownloadAction action = new ContextDownloadAction( + to, + null, + null, + null, + null, + null, + "ai", + "jsonl", + files, + true, + false + ); + + action.act(Outputter.getDefault(), pb, client); + + ArgumentCaptor captor = ArgumentCaptor.forClass(InputStream.class); + verify(files).writeToFile(eq(to.toString()), captor.capture()); + String content; + try (InputStream is = captor.getValue()) { + content = new String(is.readAllBytes(), UTF_8); + } + + // ensure only the AI-context string (id=2) is present + assertTrue(content.contains("\"id\":2")); + assertFalse(content.contains("\"id\":1")); + assertFalse(content.contains("\"id\":3")); + + verifyNoMoreInteractions(files); + } + + @Test + public void testSinceFilterExcludesStringsAndNoWrite() throws Exception { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + + var projectFull = ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId())) + .addFile("first.txt", "plain", 101L, null, null, "/%original_file_name%") + .build(); + projectFull.setType(Type.FILES_BASED); + + ProjectClient client = mock(ProjectClient.class); + when(client.downloadFullProject(null)).thenReturn(projectFull); + when(client.listLabels()).thenReturn(List.of()); + + // SourceStringBuilder.standard() has createdAt 2020-03-20, so sinceFilter 2020-03-21 will exclude it + SourceString ss = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(701L, "the-text", "manual", "the.key", 101L) + .build(); + + when(client.listSourceString(null, null, null, null, null, null, null)) + .thenReturn(Arrays.asList(ss)); + + FilesInterface files = mock(FilesInterface.class); + File to = new File("out3.jsonl"); + + ContextDownloadAction action = new ContextDownloadAction( + to, + null, + null, + null, + null, + LocalDate.of(2020, 3, 21), + null, + "jsonl", + files, + true, + false + ); + + action.act(Outputter.getDefault(), pb, client); + + // since all strings are filtered out, writeToFile should not be called + verify(files, never()).writeToFile(any(), any()); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/actions/ContextResetActionTest.java b/src/test/java/com/crowdin/cli/commands/actions/ContextResetActionTest.java new file mode 100644 index 000000000..d337c5f45 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/actions/ContextResetActionTest.java @@ -0,0 +1,115 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.ProjectBuilder; +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.client.models.SourceStringBuilder; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.properties.NewProjectPropertiesUtilBuilder; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.client.core.model.PatchRequest; +import com.crowdin.client.projectsgroups.model.Type; +import com.crowdin.client.sourcestrings.model.SourceString; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +public class ContextResetActionTest { + + @Test + public void testContextReset() { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + + // Build project with one file (id=101) + var projectFull = ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId())) + .addFile("first.txt", "plain", 101L, null, null, "/%original_file_name%") + .build(); + projectFull.setType(Type.FILES_BASED); + + ProjectClient client = mock(ProjectClient.class); + when(client.downloadFullProject(null)).thenReturn(projectFull); + when(client.listLabels()).thenReturn(List.of()); + + SourceString ss = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(701L, "the-text", "manual\n\n✨ AI Context\nai-content\n✨ 🔚", "the.key", 101L) + .build(); + + when(client.listSourceString(null, null, null, null, null, null, null)) + .thenReturn(Arrays.asList(ss)); + + ContextResetAction action = new ContextResetAction( + null, + null, + null, + null, + null, + false, + 100, + false, + false + ); + + action.act(Outputter.getDefault(), pb, client); + + verify(client).downloadFullProject(null); + verify(client).listLabels(); + verify(client).listSourceString(null, null, null, null, null, null, null); + + ArgumentCaptor captor = ArgumentCaptor.forClass(List.class); + verify(client, times(1)).batchEditSourceStrings(captor.capture()); + + List batch = captor.getValue(); + assertEquals(1, batch.size()); + PatchRequest req = (PatchRequest) batch.get(0); + assertEquals("/701/context", req.getPath()); + assertEquals("manual", req.getValue()); + } + + @Test + public void testContextResetDryrun() { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + + // Build project with one file (id=101) + var projectFull = ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId())) + .addFile("first.txt", "plain", 101L, null, null, "/%original_file_name%") + .build(); + projectFull.setType(Type.FILES_BASED); + + ProjectClient client = mock(ProjectClient.class); + when(client.downloadFullProject(null)).thenReturn(projectFull); + when(client.listLabels()).thenReturn(List.of()); + + SourceString ss = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(701L, "the-text", "manual\n\n✨ AI Context\nai-content\n✨ 🔚", "the.key", 101L) + .build(); + + when(client.listSourceString(null, null, null, null, null, null, null)) + .thenReturn(Arrays.asList(ss)); + + ContextResetAction action = new ContextResetAction( + null, + null, + null, + null, + null, + true, + 100, + false, + false + ); + + action.act(Outputter.getDefault(), pb, client); + + verify(client).downloadFullProject(null); + verify(client).listLabels(); + verify(client).listSourceString(null, null, null, null, null, null, null); + + verify(client, never()).batchEditSourceStrings(any()); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/actions/ContextStatusActionTest.java b/src/test/java/com/crowdin/cli/commands/actions/ContextStatusActionTest.java new file mode 100644 index 000000000..b2b3149db --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/actions/ContextStatusActionTest.java @@ -0,0 +1,59 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.ProjectBuilder; +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.client.models.SourceStringBuilder; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.properties.NewProjectPropertiesUtilBuilder; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.client.projectsgroups.model.Type; +import com.crowdin.client.sourcestrings.model.SourceString; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.mockito.Mockito.*; + +public class ContextStatusActionTest { + + @Test + public void testContextStatus() { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + + // Build project with one file (id=101) + var projectFull = ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId())) + .addFile("first.txt", "plain", 101L, null, null, "/%original_file_name%") + .build(); + projectFull.setType(Type.FILES_BASED); + + ProjectClient client = mock(ProjectClient.class); + when(client.downloadFullProject(null)).thenReturn(projectFull); + when(client.listLabels()).thenReturn(List.of()); + + SourceString ss = SourceStringBuilder.standard() + .setProjectId(Long.parseLong(pb.getProjectId())) + .setIdentifiers(701L, "the-text", "manual\n\n✨ AI Context\nai-content\n✨ 🔚", "the.key", 101L) + .build(); + + when(client.listSourceString(null, null, null, null, null, null, null)) + .thenReturn(Arrays.asList(ss)); + + ContextStatusAction action = new ContextStatusAction( + null, + null, + null, + null, + null, + true, + false, + false + ); + + action.act(Outputter.getDefault(), pb, client); + + verify(client).downloadFullProject(null); + verify(client).listLabels(); + verify(client).listSourceString(null, null, null, null, null, null, null); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/actions/ContextUploadActionTest.java b/src/test/java/com/crowdin/cli/commands/actions/ContextUploadActionTest.java new file mode 100644 index 000000000..39baccaa9 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/actions/ContextUploadActionTest.java @@ -0,0 +1,114 @@ +package com.crowdin.cli.commands.actions; + +import com.crowdin.cli.client.ProjectClient; +import com.crowdin.cli.commands.Outputter; +import com.crowdin.cli.properties.NewProjectPropertiesUtilBuilder; +import com.crowdin.cli.properties.ProjectProperties; +import com.crowdin.cli.utils.AiContextUtil; +import com.crowdin.client.core.model.PatchRequest; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.*; + +public class ContextUploadActionTest { + + @Test + public void testBatchesAndCallsClient() throws Exception { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + ProjectClient client = mock(ProjectClient.class); + + File file = new File("dummy.jsonl"); + + var r1 = new AiContextUtil.StringContextRecord(11L, "k1", "t1", "f", "man1", "ai1"); + var r2 = new AiContextUtil.StringContextRecord(22L, "k2", "t2", "f", "man2", ""); + var r3 = new AiContextUtil.StringContextRecord(33L, "k3", "t3", "f", "", "ai3"); + + try (var mocked = mockStatic(AiContextUtil.class)) { + mocked.when(() -> AiContextUtil.readRecords(file)).thenReturn(List.of(r1, r2, r3)); + mocked.when(() -> AiContextUtil.fullContext("man1", "ai1")).thenReturn("man1\n\n✨ AI Context\nai1\n✨ 🔚"); + mocked.when(() -> AiContextUtil.fullContext("man2", "")).thenReturn("man2"); + mocked.when(() -> AiContextUtil.fullContext("", "ai3")).thenReturn("\n\n✨ AI Context\nai3\n✨ 🔚"); + + ContextUploadAction action = new ContextUploadAction(file, false, false, false, 2); + action.act(Outputter.getDefault(), pb, client); + } + + // Should call client.batchEditSourceStrings twice: sizes 2 and 1 + ArgumentCaptor captor = ArgumentCaptor.forClass(List.class); + verify(client, times(2)).batchEditSourceStrings(captor.capture()); + + List all = captor.getAllValues(); + assertEquals(2, all.size()); + assertEquals(2, all.get(0).size()); + assertEquals(1, all.get(1).size()); + + // Check first batch contents (ids 11 and 22) + PatchRequest p1 = (PatchRequest) all.get(0).get(0); + PatchRequest p2 = (PatchRequest) all.get(0).get(1); + PatchRequest p3 = (PatchRequest) all.get(1).get(0); + + assertEquals("/11/context", p1.getPath()); + assertTrue(p1.getValue().toString().contains("man1")); + assertTrue(p1.getValue().toString().contains("ai1")); + + assertEquals("/22/context", p2.getPath()); + assertTrue(p2.getValue().toString().contains("man2")); + + assertEquals("/33/context", p3.getPath()); + assertTrue(p3.getValue().toString().contains("ai3")); + } + + @Test + public void testOverwriteFiltersRecords() throws Exception { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + ProjectClient client = mock(ProjectClient.class); + + File file = new File("dummy_overwrite.jsonl"); + + var keep = new AiContextUtil.StringContextRecord(10L, "k10", "t10", "f", "man10", ""); + var skip = new AiContextUtil.StringContextRecord(11L, "k11", "t11", "f", "man11", "ai11"); + + try (var mocked = mockStatic(AiContextUtil.class)) { + mocked.when(() -> AiContextUtil.readRecords(file)).thenReturn(List.of(keep, skip)); + mocked.when(() -> AiContextUtil.fullContext("man10", "")).thenReturn("man10"); + mocked.when(() -> AiContextUtil.fullContext("man11", "ai11")).thenReturn("man11\n\n✨ AI Context\nai11\n✨ 🔚"); + + // overwrite = true should KEEP only records with empty ai_context (i.e., 'keep') + ContextUploadAction action = new ContextUploadAction(file, true, false, false, 10); + action.act(Outputter.getDefault(), pb, client); + } + + ArgumentCaptor captor = ArgumentCaptor.forClass(List.class); + verify(client, times(1)).batchEditSourceStrings(captor.capture()); + List batch = captor.getValue(); + assertEquals(1, batch.size()); + PatchRequest req = (PatchRequest) batch.get(0); + assertEquals("/10/context", req.getPath()); + } + + @Test + public void testDryRunDoesNotCallClient() throws Exception { + ProjectProperties pb = NewProjectPropertiesUtilBuilder.minimalBuilt().build(); + ProjectClient client = mock(ProjectClient.class); + + File file = new File("dummy_dry.jsonl"); + + var rec = new AiContextUtil.StringContextRecord(21L, "k21", "t21", "f", "man21", "ai21"); + + try (var mocked = mockStatic(AiContextUtil.class)) { + mocked.when(() -> AiContextUtil.readRecords(file)).thenReturn(List.of(rec)); + mocked.when(() -> AiContextUtil.fullContext("man21", "ai21")).thenReturn("man21\n\n✨ AI Context\nai21\n✨ 🔚"); + + ContextUploadAction action = new ContextUploadAction(file, false, true, false, 10); + action.act(Outputter.getDefault(), pb, client); + } + + verify(client, never()).batchEditSourceStrings(any()); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/picocli/ContextDownloadSubcommandTest.java b/src/test/java/com/crowdin/cli/commands/picocli/ContextDownloadSubcommandTest.java new file mode 100644 index 000000000..0b9ed9d13 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/picocli/ContextDownloadSubcommandTest.java @@ -0,0 +1,53 @@ +package com.crowdin.cli.commands.picocli; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.stream.Stream; + +import static com.crowdin.cli.commands.picocli.GenericCommand.RESOURCE_BUNDLE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.verify; + +public class ContextDownloadSubcommandTest extends PicocliTestUtils { + + @Test + public void testContextDownload() { + this.execute(CommandNames.CONTEXT, CommandNames.CONTEXT_DOWNLOAD, "some/path/to/file.jsonl"); + verify(actionsMock).contextDownload(any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean()); + this.check(true); + } + + @Test + public void testContextDownloadInvalid() { + this.executeInvalidParams(CommandNames.CONTEXT, CommandNames.CONTEXT_DOWNLOAD); + } + + @ParameterizedTest + @MethodSource + public void testContextDownloadInvalidOptions(String since, String format, String status, List expErrors) { + ContextDownloadSubcommand subcommand = new ContextDownloadSubcommand(); + subcommand.since = since; + subcommand.format = format; + subcommand.status = status; + + List errors = subcommand.checkOptions(); + assertThat(errors, Matchers.equalTo(expErrors)); + } + + public static Stream testContextDownloadInvalidOptions() { + return Stream.of( + arguments("2020-12-01", "jsonl", "ai", List.of()), + arguments("invalid", "text", "invalid", List.of(RESOURCE_BUNDLE.getString("error.context.invalid_since"), RESOURCE_BUNDLE.getString("error.context.invalid_format"), RESOURCE_BUNDLE.getString("error.context.invalid_status"))), + arguments("invalid", "text", null, List.of(RESOURCE_BUNDLE.getString("error.context.invalid_since"), RESOURCE_BUNDLE.getString("error.context.invalid_format"))), + arguments(null, "text", null, List.of(RESOURCE_BUNDLE.getString("error.context.invalid_format"))) + ); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/picocli/ContextResetSubcommandTest.java b/src/test/java/com/crowdin/cli/commands/picocli/ContextResetSubcommandTest.java new file mode 100644 index 000000000..b39c17c61 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/picocli/ContextResetSubcommandTest.java @@ -0,0 +1,21 @@ +package com.crowdin.cli.commands.picocli; + +import org.junit.jupiter.api.Test; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.verify; + +public class ContextResetSubcommandTest extends PicocliTestUtils { + + @Test + public void testContextReset() { + this.execute(CommandNames.CONTEXT, CommandNames.CONTEXT_RESET, "--all"); + verify(actionsMock).contextReset(any(), any(), any(), any(), any(), anyBoolean(), anyInt(), anyBoolean(), anyBoolean()); + this.check(true); + } + + @Test + public void testContextResetInvalid() { + this.executeInvalidParams(CommandNames.CONTEXT, CommandNames.CONTEXT_RESET); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/picocli/ContextStatusSubcommandTest.java b/src/test/java/com/crowdin/cli/commands/picocli/ContextStatusSubcommandTest.java new file mode 100644 index 000000000..2f2bc8247 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/picocli/ContextStatusSubcommandTest.java @@ -0,0 +1,17 @@ +package com.crowdin.cli.commands.picocli; + +import org.junit.jupiter.api.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.verify; + +public class ContextStatusSubcommandTest extends PicocliTestUtils { + + @Test + public void testContextStatus() { + this.execute(CommandNames.CONTEXT, CommandNames.CONTEXT_STATUS); + verify(actionsMock).contextStatus(any(), any(), any(), any(), any(), anyBoolean(), anyBoolean(), anyBoolean()); + this.check(true); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/picocli/ContextUploadSubcommandTest.java b/src/test/java/com/crowdin/cli/commands/picocli/ContextUploadSubcommandTest.java new file mode 100644 index 000000000..2e65aa6e7 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/picocli/ContextUploadSubcommandTest.java @@ -0,0 +1,11 @@ +package com.crowdin.cli.commands.picocli; + +import org.junit.jupiter.api.Test; + +public class ContextUploadSubcommandTest extends PicocliTestUtils { + + @Test + public void testContextUploadInvalid() { + this.executeInvalidParams(CommandNames.CONTEXT, CommandNames.CONTEXT_UPLOAD); + } +} diff --git a/src/test/java/com/crowdin/cli/commands/picocli/PicocliTestUtils.java b/src/test/java/com/crowdin/cli/commands/picocli/PicocliTestUtils.java index e1d03238b..0a83065fa 100644 --- a/src/test/java/com/crowdin/cli/commands/picocli/PicocliTestUtils.java +++ b/src/test/java/com/crowdin/cli/commands/picocli/PicocliTestUtils.java @@ -12,6 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; public class PicocliTestUtils { @@ -156,6 +157,12 @@ void mockActions() { when(actionsMock.listApps(anyBoolean())).thenReturn(actionMock); when(actionsMock.uninstallApp(anyString(), anyBoolean())).thenReturn(actionMock); when(actionsMock.installApp(anyString())).thenReturn(actionMock); + when(actionsMock.contextDownload(any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean())) + .thenReturn(actionMock); + when(actionsMock.contextReset(any(), any(), any(), any(), any(), anyBoolean(), anyInt(), anyBoolean(), anyBoolean())) + .thenReturn(actionMock); + when(actionsMock.contextStatus(any(), any(), any(), any(), any(), anyBoolean(), anyBoolean(), anyBoolean())) + .thenReturn(actionMock); } private void mockBuilders() { diff --git a/src/test/java/com/crowdin/cli/utils/AiContextUtilTest.java b/src/test/java/com/crowdin/cli/utils/AiContextUtilTest.java new file mode 100644 index 000000000..f0b2edddd --- /dev/null +++ b/src/test/java/com/crowdin/cli/utils/AiContextUtilTest.java @@ -0,0 +1,40 @@ +package com.crowdin.cli.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class AiContextUtilTest { + + @Test + public void testGetManualContext() { + assertEquals( + "This is the manual context.", + AiContextUtil.getManualContext("This is the manual context.\n\n✨ AI Context\nThis is the AI context.\n✨ 🔚") + ); + assertEquals( + "This is the manual context.", + AiContextUtil.getManualContext("This is the manual context.") + ); + assertEquals( + "", + AiContextUtil.getManualContext("") + ); + } + + @Test + public void testGetAiContextSection() { + assertEquals( + "This is the AI context.", + AiContextUtil.getAiContextSection("This is the manual context.\n\n✨ AI Context\nThis is the AI context.\n✨ 🔚") + ); + assertEquals( + "", + AiContextUtil.getAiContextSection("This is the manual context.") + ); + assertEquals( + "", + AiContextUtil.getAiContextSection("") + ); + } +} diff --git a/src/test/java/com/crowdin/cli/utils/GlobUtilTest.java b/src/test/java/com/crowdin/cli/utils/GlobUtilTest.java new file mode 100644 index 000000000..354e2f2b9 --- /dev/null +++ b/src/test/java/com/crowdin/cli/utils/GlobUtilTest.java @@ -0,0 +1,25 @@ +package com.crowdin.cli.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class GlobUtilTest { + + @Test + public void testGlobToRegex() { + assertTrue(GlobUtil.matches("*.txt", "file.txt")); + assertFalse(GlobUtil.matches("*.txt", "file.jpg")); + assertTrue(GlobUtil.matches("**/*.txt", "dir/file.txt")); + assertFalse(GlobUtil.matches("**/*.txt", "dir/file.jpg")); + assertTrue(GlobUtil.matches("file?.txt", "file1.txt")); + assertFalse(GlobUtil.matches("file?.txt", "file12.txt")); + assertTrue(GlobUtil.matches("file[0-9].txt", "file1.txt")); + assertFalse(GlobUtil.matches("file[0-9].txt", "filea.txt")); + assertTrue(GlobUtil.matches("src/**/test.json", "src/a/n/c/test.json")); + assertTrue(GlobUtil.matches("**/*.*", "src/a/n/c/test.json")); + assertTrue(GlobUtil.matches("src/a/b.txt", "src/a/b.txt")); + assertFalse(GlobUtil.matches("src/a/b.txt", "src/a/b.json")); + } +} diff --git a/website/mantemplates/crowdin-context-download.adoc b/website/mantemplates/crowdin-context-download.adoc new file mode 100644 index 000000000..2d4e54111 --- /dev/null +++ b/website/mantemplates/crowdin-context-download.adoc @@ -0,0 +1,16 @@ +:includedir: ../generated-picocli-docs +:command: crowdin-context-download + +== crowdin context download + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-description] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-synopsis] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-arguments] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-commands] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-options] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-footer] diff --git a/website/mantemplates/crowdin-context-reset.adoc b/website/mantemplates/crowdin-context-reset.adoc new file mode 100644 index 000000000..ec2c8cf1a --- /dev/null +++ b/website/mantemplates/crowdin-context-reset.adoc @@ -0,0 +1,16 @@ +:includedir: ../generated-picocli-docs +:command: crowdin-context-reset + +== crowdin context reset + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-description] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-synopsis] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-arguments] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-commands] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-options] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-footer] diff --git a/website/mantemplates/crowdin-context-status.adoc b/website/mantemplates/crowdin-context-status.adoc new file mode 100644 index 000000000..82ff7d739 --- /dev/null +++ b/website/mantemplates/crowdin-context-status.adoc @@ -0,0 +1,16 @@ +:includedir: ../generated-picocli-docs +:command: crowdin-context-status + +== crowdin context status + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-description] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-synopsis] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-arguments] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-commands] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-options] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-footer] diff --git a/website/mantemplates/crowdin-context-upload.adoc b/website/mantemplates/crowdin-context-upload.adoc new file mode 100644 index 000000000..da337c4e5 --- /dev/null +++ b/website/mantemplates/crowdin-context-upload.adoc @@ -0,0 +1,16 @@ +:includedir: ../generated-picocli-docs +:command: crowdin-context-upload + +== crowdin context upload + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-description] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-synopsis] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-arguments] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-commands] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-options] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-footer] diff --git a/website/mantemplates/crowdin-context.adoc b/website/mantemplates/crowdin-context.adoc new file mode 100644 index 000000000..2f5049f6a --- /dev/null +++ b/website/mantemplates/crowdin-context.adoc @@ -0,0 +1,16 @@ +:includedir: ../generated-picocli-docs +:command: crowdin-context + +== crowdin context + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-description] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-synopsis] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-arguments] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-commands] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-options] + +include::{includedir}/{command}.adoc[tag=picocli-generated-man-section-footer] diff --git a/website/sidebars.js b/website/sidebars.js index cf2aa6018..ffa886caf 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -300,7 +300,23 @@ const sidebars = { 'commands/crowdin-app-install', 'commands/crowdin-app-uninstall', ] - } + }, + { + type: 'category', + label: 'crowdin context', + link: { + type: 'doc', + id: 'commands/crowdin-context' + }, + collapsible: true, + collapsed: true, + items: [ + 'commands/crowdin-context-download', + 'commands/crowdin-context-upload', + 'commands/crowdin-context-reset', + 'commands/crowdin-context-status', + ] + }, ], }, 'ci-cd',