From ed484c5a72ec817f704a43ff75c62fd78a8306bb Mon Sep 17 00:00:00 2001 From: Martin Price Date: Thu, 27 Jun 2024 08:54:07 +0100 Subject: [PATCH 01/15] Progress so far on download functions --- commands/download.bee.inc | 819 +++++++++++++++++++++++++++----------- includes/globals.inc | 5 + 2 files changed, 582 insertions(+), 242 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 540f5ffd..8859acd4 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -9,6 +9,18 @@ */ define('BEE_USERAGENT', 'Bee - the command line tool for Backdrop CMS'); +/** + * The API URL used for GitHub API calls. + */ +define('BEE_GITHUB_API_URL', 'https://api.github.com/'); + +/** + * Include the common functions file so we can re-use some Backdrop functions + * for handling .info files and dependencies. + */ +global $_bee_backdrop_root; +require_once $_bee_backdrop_root . '/core/includes/common.inc'; + /** * Implements hook_bee_command(). */ @@ -90,7 +102,7 @@ function download_bee_callback($arguments, $options) { $api_call_estimate = count($arguments['projects']) * 3; // Check GitHub API quota to ensure we can complete. - if (!check_github_api_quota($github_api_token, $api_call_estimate)) { + if (!download_bee_check_github_api_quota($github_api_token, $api_call_estimate)) { return; } @@ -122,23 +134,15 @@ function download_bee_callback($arguments, $options) { while ($project_count < count($arguments['projects'])) { $project = $arguments['projects'][$project_count]; // Check if the project exists by trying to get the repo homepage. - $organisation = 'backdrop-contrib'; - $url = "https://github.com/$organisation/$project"; - $headers = get_headers($url); - if (!$headers) { - bee_message(bt("Unable to connect to !url.", array( - '!url' => $url, - )), 'error'); - continue; - } - $response = substr($headers[0], 9, 3); - if ($response >= 400) { + $organization = 'backdrop-contrib'; + $url = "https://github.com/$organization/$project"; + if (!download_bee_check_url_exists($url)) { bee_message(bt("The '!project' project repository could not be found. Please check your spelling and try again.", array( '!project' => $project, )), 'error'); } else { - $info = download_bee_git_info($project, $github_api_token); + $info = download_bee_get_project_info($project, $organization, '', '', $github_api_token); if (empty($info)) { // If getting the info file has failed, show an error message here and // exit. @@ -150,18 +154,32 @@ function download_bee_callback($arguments, $options) { // Get the list of dependencies and add to list of projects. if (!empty($info['dependencies'])) { $dependencies = $info['dependencies']; - foreach ($dependencies as $dependency) { + foreach ($dependencies as $dependency_detail) { + $submodule = ''; // Remove any minimum version requirements to get just the project // name. - $dependency = explode(" ", $dependency, 2); - $dependency = $dependency[0]; + // $dependency = explode(" ", $dependency, 2); + // $dependency = $dependency[0]; + // Check whether the dependency is a submodule. + if (isset($dependency_detail['project'])) { + // If so, get the submodule. + $submodule = $dependency_detail['name']; + // Get the parent module. + $dependency = $dependency_detail['project']; + } + else { + // If not, get the module name. + $dependency = $dependency_detail['name']; + } + // For now, ignoring the version part of dependencies as in most + // cases we will be getting the latest versions. // Check if the dependency is a core project. $dependency_is_core = isset($core_projects[$dependency]); // Check if dependency exists in the site file system. Only // check contrib dependencies. if (!$dependency_is_core) { // Get information about the dependency project. - $dependency_info = download_bee_git_info($dependency, $github_api_token); + $dependency_info = download_bee_get_project_info($dependency, $organization, '', '', $github_api_token); if (empty($dependency_info)) { // If getting the info file has failed, show an error message here // and exit. @@ -182,8 +200,8 @@ function download_bee_callback($arguments, $options) { // If project is already in the list to download, do not add to // the list to download again. bee_message(bt("The '!dependency' !dependency_type is also required by the '!project' !project_type.", array( - '!dependency' => $dependency, - '!dependency_type' => $dependency_info['type'], + '!dependency' => !empty($submodule) ? "$dependency:$submodule" : $dependency, + '!dependency_type' => (!empty($submodule) ? 'sub' : '') . $dependency_info['type'], '!project' => $project, '!project_type' => $info['type'], )), 'status'); @@ -193,8 +211,8 @@ function download_bee_callback($arguments, $options) { // and prepare message. $arguments['projects'][] = $dependency; bee_message(bt("The '!dependency' !dependency_type will also be downloaded, as it is required by the '!project' !project_type.", array( - '!dependency' => $dependency, - '!dependency_type' => $dependency_info['type'], + '!dependency' => !empty($submodule) ? "$dependency:$submodule" : $dependency, + '!dependency_type' => (!empty($submodule) ? 'sub' : '') . $dependency_info['type'], '!project' => $project, '!project_type' => $info['type'], )), 'status'); @@ -223,26 +241,9 @@ function download_bee_callback($arguments, $options) { )), 'error'); return; } - else { - // Add an 's' to the end of the type name. - $type_folder = $info['type'] . 's'; - } - // Get the directory to download the project into. - if (!empty($_bee_backdrop_site)) { - $destination = "$_bee_backdrop_root/sites/$_bee_backdrop_site/" . $type_folder; - } - elseif (!empty($_bee_backdrop_root)) { - $destination = "$_bee_backdrop_root/" . $type_folder; - } - else { - bee_message(bt("The download destination could not be determined. Re-run the command from within a Backdrop installation, or set the global '--root'/'--site' options."), 'error'); - return; - } - if (file_exists("$destination/contrib")) { - $destination .= '/contrib'; - } - $destination .= "/$project"; + $destination = download_bee_get_destination_path($project, $info['type']); + bee_instant_message('$destination:' . $destination, 'debug'); // Check if the project exists within the site file system. $project_existing_location = download_bee_check_project_exists($project, $info['type']); $project_allow_multisite_copy = download_bee_check_multisite_copy($allow_multisite_copy, $project_existing_location); @@ -261,7 +262,7 @@ function download_bee_callback($arguments, $options) { } // Download the project. - if (download_bee_download_project($project, $info, $destination)) { + if (download_bee_download_project($project, $info['download_url'], $destination, $info['branch'])) { bee_message(bt("'!project' was downloaded into '!directory'.", array( '!project' => $project, '!directory' => $destination, @@ -283,11 +284,11 @@ function download_core_bee_callback($arguments, $options) { $api_call_estimate = 1; // Check GitHub API quota to ensure we can complete. - if (!check_github_api_quota($github_api_token, $api_call_estimate)) { + if (!download_bee_check_github_api_quota($github_api_token, $api_call_estimate)) { return; } - $info = download_bee_git_info('backdrop', $github_api_token); + $info = download_bee_get_project_info('backdrop', 'backdrop', '', '', $github_api_token); // Get or create the directory to download Backdrop into. $destination = !empty($arguments['directory']) ? $arguments['directory'] : getcwd(); @@ -308,7 +309,7 @@ function download_core_bee_callback($arguments, $options) { } // Download Backdrop. - if (download_bee_download_project('backdrop', $info, $destination)) { + if (download_bee_download_project('backdrop', $info['download_url'], $destination, $info['branch'])) { bee_message(bt("Backdrop was downloaded into '!directory'.", array( '!directory' => $destination, )), 'success'); @@ -316,155 +317,323 @@ function download_core_bee_callback($arguments, $options) { } /** - * Get information about a project from GitHub. + * Get required information about the project. * * @param string $project * The name of the project. + * @param string $organization + * The name of the GitHub organization. + * @param string $release + * A string relating to a release option. Defaults to empty string (''): + * - 'dev': Download the dev version from the default branch. + * - 'branch': Download the dev version from an alternative branch. Specified + * using the $branch parameter. + * - 'latest': Downloads the latest release that is neither a draft nor a + * pre-release. This will be the default anyway if the parameter is left + * blank. If there is not a valid latest release, the dev branch will be + * offered (see 'dev'). + * - '1.x-1.2.3': The tag of a given release. If the release cannot be found + * an error message will be given and the function will reu. + * - 'select': A list of valid options will be offered including: + * - dev: (see dev). + * - All releases that are not draft. Latest will be labelled and + * pre-releases will also be labelled. + * @param string $branch + * A string containing the name of the branch. Defaults to empty string (''). * @param string $github_api_token - * The GitHub API Personal Access Token to apply to API calls. + * The GitHub API Personal Access Token to apply to API calls. Defaults to + * empty string (''). * - * @return array - * An associative array of information about the project, possibly containing: - * - url: The URL to download the project. - * - branch: The default branch of the project. - * - type: The 'type' of project (e.g. module, theme or layout). + * @return array|false + * An associative array of information about the project, including all + * relevant information: + * - 'download_url': The URL to download the project. Always included if + * request is successful. + * - 'type': the project type (i.e. 'module', 'theme' or 'layout'). Always + * included if request is successful. + * - 'branch': The branch of the project to be downloaded. Included if the + * default or another branch is used. Defaults to blank string if not used. + * - 'release': Associative array of key release information: + * - 'tag_name': the tag name selected. + * - 'published_at': the date the release is published. + * - 'release_url': the URL of the release page on GitHub. + * Included if a release is specified or selected from the select screen. + * - 'dependencies': An array of dependency lines. Defaults to blank array if + * no dependencies. */ -function download_bee_git_info($project, $github_api_token = '') { - $info = array(); - $organisation = ($project == 'backdrop') ? 'backdrop' : 'backdrop-contrib'; - $api_url = 'https://api.github.com/repos'; - - // Get the download URL of the latest release. - $url = "https://github.com/$organisation/$project/releases/latest/download/$project.zip"; - $headers = get_headers($url); - if (!$headers) { - bee_message(bt("Unable to connect to !url.", array( - '!url' => $url, - )), 'error'); - return; - } - $response = substr($headers[0], 9, 3); - if ($response < 400) { - $info['url'] = $url; - } - else { - // Offer to get the dev version instead. - $answer = bee_confirm(bt("There is no release for '!organisation/!project'. Do you you want to download the dev version instead?", array( - '!organisation' => $organisation, - '!project' => $project, - ))); +function download_bee_get_project_info($project, $organization, $release = '', $branch = '', $github_api_token = '') { - if ($answer) { - $repo_info = FALSE; - // Check GitHub API quota to ensure we can continue. - if (!check_github_api_quota($github_api_token)) { - return; + // Define the base for all GitHub API calls we may use in this function. + $endpoint_base = "repos/$organization/$project"; + + // Select the actions based on the release information entered. + switch ($release) { + case 'latest': + // If 'latest' is explicitly entered, it is the same as if release is + // left blank. + case '': + $preferred_option = 'latest'; + // Check that a valid latest release exists. For GitHub to count a + // release as 'latest' it must be published (i.e. not draft) and not a + // pre-release. + $defined_release_url = "https://github.com/$organization/$project/releases/latest"; + if (download_bee_check_url_exists($defined_release_url)) { + $final_option = 'latest'; } - if ($curl_handle = curl_init("$api_url/$organisation/$project")) { - // Prepare headers for curl request. - $curlopt_httpheader = array(); - $curlopt_httpheader[] = 'Content-Type: application/json'; - // If a token has been passed, add this to the header. - if (!empty($github_api_token)) { - $curlopt_httpheader[] = "Authorization: token $github_api_token"; + else { + // If the URL does not exist, offer to get the dev version instead. + $choice = bee_confirm(bt("There is no valid latest release for '!organization/!project'. Do you want to download the dev version instead?", array( + '!organization' => $organization, + '!project' => $project, + )), TRUE); + if ($choice) { + // If 'Yes' mode was enabled or user responded 'Yes', then continue + // with dev version. + $final_option = 'dev'; + } + else { + // If 'Yes' mode was not enabled and user responded 'No', then provide + // a message and return FALSE. + bee_message(bt("Download operation is cancelled."), 'info'); + return FALSE; } - curl_setopt($curl_handle, CURLOPT_HTTPHEADER, $curlopt_httpheader); - curl_setopt($curl_handle, CURLOPT_USERAGENT, BEE_USERAGENT); - curl_setopt($curl_handle, CURLOPT_HEADER, 0); - curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); - $repo_info = curl_exec($curl_handle); - curl_close($curl_handle); - $repo_info = json_decode($repo_info, TRUE); } - if (!is_array($repo_info)) { - bee_message(bt('Unable to fetch repo information.'), 'error'); - return; + break; + case 'dev': + $preferred_option = 'dev'; + $final_option = 'dev'; + break; + case 'branch': + $preferred_option = 'branch'; + // Check that a branch has been specified and that it exists. + $branch_specified = !empty($branch); + if ($branch_specified ) { + $branch_url = "https://github.com/$organization/$project/tree/$branch"; + $check_branch_url = download_bee_check_url_exists($branch_url); } - // Get the download URL of the dev version. - $branch = $repo_info['default_branch']; - $url = "https://github.com/$organisation/$project/archive/$branch.zip"; - $headers = get_headers($url); - if (!$headers) { - bee_message(bt("Unable to connect to !url.", array( - '!url' => $url, - )), 'error'); - return; + if ($branch_specified && $check_branch_url) { + // If both conditions are satisfied, set the final option. + $final_option = 'branch'; } - $response = substr($headers[0], 9, 3); - if ($response < 400) { - $info['url'] = $url; - $info['branch'] = $branch; + else { + // If the branch is not specified or does not exist, offer to get the + // dev version of the default branch instead. + $choice = bee_confirm(bt("The branch you have specified (!branch) is not valid for '!organization/!project'. Do you want to download the dev version of the default branch instead?", array( + '!branch' => $branch, + '!organization' => $organization, + '!project' => $project, + )), TRUE); + if ($choice) { + // If 'Yes' mode was enabled or user responded 'Yes', then continue + // with dev version. + $final_option = 'dev'; + } + else { + // If 'Yes' mode was not enabled and user responded 'No', then provide + // a message and return FALSE. + bee_message(bt("Download operation is cancelled."), 'info'); + return FALSE; + } } - } - } - - // Get the 'type' and any dependencies of project. - if ($project != 'backdrop') { - $files = FALSE; - // Check GitHub API quota to ensure we can continue. - if (!check_github_api_quota($github_api_token)) { - return; - } - if ($curl_handle = curl_init("$api_url/$organisation/$project/contents")) { - // If a token has been passed, add this to the header. - if (!empty($github_api_token)) { - curl_setopt($curl_handle, CURLOPT_HTTPHEADER, array( - 'Content-Type: application/json', - "Authorization: token $github_api_token", - )); + case 'select': + $preferred_option = 'select'; + // Create the array for options and hardcode the 'dev' option. + $select_options = array( + 'dev' => bt("Dev - Download the current default branch."), + ); + // Check GitHub API quota to ensure we can continue. + if (!download_bee_check_github_api_quota($github_api_token)) { + return FALSE; } - curl_setopt($curl_handle, CURLOPT_USERAGENT, BEE_USERAGENT); - curl_setopt($curl_handle, CURLOPT_HEADER, 0); - curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); - $files = curl_exec($curl_handle); - curl_close($curl_handle); - $files = json_decode($files, TRUE); - } - if (!is_array($files)) { - bee_message(bt('Unable to fetch file information.'), 'error'); - return; - } - foreach ($files as $file) { - if ($file['type'] == 'file' && preg_match('/\.info$/i', $file['name'])) { - // Check GitHub API quota to ensure we can continue. - if (!check_github_api_quota($github_api_token)) { - return; + // Get the list of releases. + $endpoint = "$endpoint_base/releases"; + $releases = download_bee_github_api_call($endpoint, $github_api_token); + // If one or more releases exist, then loop through them to get more + // information. + if (!empty($releases)) { + foreach ($releases as $release_data) { + // Convert the array to being an associative array keyed by tag_name. + $releases_by_tag[$release_data['tag_name']] = $release_data; + // If the release is not a prerelease, also put into an array to check + // for latest release. + if (!$release_data['prerelease']) { + $full_release_list[] = $release_data['tag_name']; + } } - $curl_handle = curl_init("$api_url/$organisation/$project/contents/" . $file['name']); - // Prepare headers for curl request. - $curlopt_httpheader = array(); - $curlopt_httpheader[] = 'Accept: application/vnd.github.v3.raw'; - // If a token has been passed, add this to the header. - if (!empty($github_api_token)) { - $curlopt_httpheader[] = "Authorization: token $github_api_token"; + // If full releases exist, find the latest release. + if (!empty($full_release_list)) { + usort($full_release_list, 'version_compare'); + $latest_release_tag = end($full_release_list); + $default = $latest_release_tag; } - curl_setopt($curl_handle, CURLOPT_HTTPHEADER, $curlopt_httpheader); - curl_setopt($curl_handle, CURLOPT_USERAGENT, BEE_USERAGENT); - curl_setopt($curl_handle, CURLOPT_HEADER, 0); - curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); - - $info_file = curl_exec($curl_handle); - curl_close($curl_handle); - $lines = explode("\n", $info_file); - // Declare dependencies as array. - $info['dependencies'] = array(); - foreach ($lines as $line) { - $values = explode('=', $line); - // Get the type of project. - if (trim($values[0]) == 'type') { - $info['type'] = trim($values[1]); + else { + $latest_release_tag = NULL; + $default = 'dev'; + } + // Add each release to the list of options. + foreach ($releases_by_tag as $key => $release_data) { + // Define any suffixes needed for 'latest' or 'pre-release'. + $suffix = ''; + if ($key == $latest_release_tag) { + $suffix = bt(" (latest)"); } - // Get any dependencies of project. - if (trim($values[0]) == 'dependencies[]') { - $info['dependencies'][] = trim($values[1]); + if ($release_data['prerelease']) { + $suffix = bt(" (pre-release)"); } + $select_options[$key] = $key . $suffix; } - // Exit loop as only need to check .info file. - break; + // Set the message to be used. + $message = bt("Select the version you wish to download."); + } + else { + // If no releases exist, tailor the message and default. + $message = bt("No releases exist but you can download the dev version."); + $default = 'dev'; + } + // Present the choice to the user. + $selection = bee_choice($select_options, $message, $default); + // Use the selection to set variables that will be used in the next step. + if ($selection == 'dev') { + $final_option = 'dev'; + } + else { + $final_option = 'defined_release'; + $defined_release = $selection; + } + break; + default: + // If user has not entered a keyword such as 'dev', 'latest' or 'select' + // then, we interpret that as being a release tag and check for it. + $preferred_option = 'defined_release'; + $defined_release = $release; + $defined_release_url = "https://github.com/$organization/$project/releases/tag/$defined_release"; + if (download_bee_check_url_exists($defined_release_url)) { + // If the release page exists, then this entry is valid and we can set + // the final option. + $final_option = 'defined_release'; + } + else { + // If the release page does not exist offer to get the dev version + // instead. + $choice = bee_confirm(bt("The release you have specified (!release_tag) for '!organization/!project' does not exist. Do you want to download the dev version instead?", array( + '!release_tag' => $defined_release, + '!organization' => $organization, + '!project' => $project, + )), TRUE); + if ($choice) { + // If 'Yes' mode was enabled or user responded 'Yes', then continue + // with dev version. + $final_option = 'dev'; + } + else { + // If 'Yes' mode was not enabled and user responded 'No', then provide + // a message and return FALSE. + bee_message(bt("Download operation is cancelled."), 'info'); + return FALSE; + } + } + break; + } + bee_instant_message('$final_option:' . $final_option, 'debug'); + // Process the final option. + switch ($final_option) { + case 'latest': + // $download_url = "https://github.com/$organization/$project/releases/latest/download/$project.zip"; + // Check GitHub API quota to ensure we can continue. + if (!download_bee_check_github_api_quota($github_api_token)) { + return FALSE; + } + // Get the latest release information. + $endpoint = "$endpoint_base/releases/latest"; + $latest_release = download_bee_github_api_call($endpoint, $github_api_token); + $defined_release = $latest_release['tag_name']; + bee_instant_message('$defined_release:' . $defined_release, 'debug'); + $defined_release_published_date = $latest_release['published_at']; + $defined_release_url = $latest_release['html_url']; + $download_url = "https://github.com/$organization/$project/releases/download/$defined_release/$project.zip"; + bee_instant_message('$download_url:' . $download_url, 'debug'); + $release_detail = array( + 'tag_name' => $defined_release, + 'published_at' => $defined_release_published_date, + 'release_url' => $defined_release_url, + ); + break; + case 'dev': + // Check GitHub API quota to ensure we can continue. + if (!download_bee_check_github_api_quota($github_api_token)) { + return FALSE; } + // Get the information about the default branch. + $endpoint = "$endpoint_base"; + $repo_info = download_bee_github_api_call($endpoint, $github_api_token); + // Check the results of the API call. + if (!$repo_info) { + // If information can't be retrieved, display an error and return + // FALSE. + bee_message(bt("Information about '!project' could not be retrieved. Please try again later.", array( + '!project' => $project, + )), 'error'); + return FALSE; + } + else { + $branch = $repo_info['default_branch']; + $download_url = "https://github.com/$organization/$project/archive/$branch.zip"; + } + break; + case 'branch': + $download_url = "https://github.com/$organization/$project/archive/$branch.zip"; + break; + case 'defined_release': + $download_url = "https://github.com/$organization/$project/releases/download/$defined_release/$project.zip"; + $release_detail = array( + 'tag_name' => $defined_release, + 'published_at' => $defined_release_published_date, + 'release_url' => $defined_release_url, + ); + break; + } + + // Start to populate the info array. + $info = array( + 'download_url' => $download_url, + 'dependencies' => array(), + ); + + // Determine the endpoint reference by whether a release or a branch, and + // populate the info array with conditional elements. + switch ($final_option) { + case 'dev': + case 'branch': + $ref = $branch; + $info['branch'] = $branch; + break; + default: + $ref = $defined_release; + $info['release'] = $release_detail; + $info['branch'] = ''; + } + bee_instant_message('$ref:' . $ref, 'debug'); + // Download the info file to get project type and dependencies. + // Check GitHub API quota to ensure we can continue. + if (!download_bee_check_github_api_quota($github_api_token)) { + return FALSE; + } + // Compile the endpoint. + $endpoint = "$endpoint_base/contents/$project.info?ref=$ref"; + bee_instant_message('$endpoint:' . $endpoint, 'debug'); + // // Retrieve the info file as an array of lines. + $info_file = download_bee_github_api_call($endpoint, $github_api_token, 'raw'); + $info_parsed = backdrop_parse_info_format($info_file[0]); + bee_instant_message(bt('The info file data:'), 'debug', $info_parsed); + $info['type'] = $info_parsed['type']; + if (isset($info_parsed['dependencies'])) { + foreach ($info_parsed['dependencies'] as $dependency) { + $dependency = backdrop_parse_dependency($dependency); + $info['dependencies'][] = $dependency; } } + // Return the info array to the calling function. return $info; } @@ -473,38 +642,44 @@ function download_bee_git_info($project, $github_api_token = '') { * * @param string $project * The name of the project to download. - * @param array $info - * An associative array of information about the project from GitHub. It - * should contain: - * - url: The URL to download the project. - * - branch: The default branch of the project. Needed when downloading the - * dev version - * - type: The 'type' of project (e.g. module, theme or layout). Needed for - * contrib projects. + * @param string $source_url + * The URL to download the project from. * @param string $destination * The path to the destination directory. + * @param string $branch + * If a dev version is being downloaded, branch is needed for the directory + * of the unzipped project. Defaults to empty string. + * @param bool $replace + * Specify whether the download replaces an existing version. Defaults to + * FALSE. + * @param bool $backup + * Specify, if replacing an existing version, whether or not to backup the + * existing code base. * * @return boolean * TRUE if the project was downloaded successfully, FALSE if not. */ -function download_bee_download_project($project, array $info, $destination) { +function download_bee_download_project($project, $source_url, $destination, $branch = '', bool $replace = FALSE, bool $backup = TRUE) { // Get a temp directory. if (!$temp = bee_get_temp($project)) { bee_message(bt('Failed to get temp directory.'), 'error'); return FALSE; } - + if (file_exists($temp)) { + bee_instant_message('$temp:' . $temp, 'debug'); + } // Get the download URL. - if (empty($url = $info['url'])) { + if (empty($url = $source_url)) { bee_message(bt("The download URL for '!project' could not be found.", array( '!project' => $project, )), 'error'); return FALSE; } - + bee_instant_message('$url:' . $url, 'debug'); // Download & extract the project. $file = "$temp/$project.zip"; - $directory = !empty($info['branch']) ? $project . '-' . $info['branch'] : $project; + $directory = !empty($branch) ? $project . '-' . $branch : $project; + bee_instant_message('$directory:' . $directory, 'debug'); $file_handle = fopen($file, 'w'); $curl_handle = curl_init($url); @@ -515,6 +690,9 @@ function download_bee_download_project($project, array $info, $destination) { curl_close($curl_handle); fclose($file_handle); + if (file_exists($file)) { + bee_instant_message('$file:' . $file, 'debug'); + } $zip = new ZipArchive; if ($zip->open($file)) { $zip->extractTo($temp); @@ -524,9 +702,44 @@ function download_bee_download_project($project, array $info, $destination) { bee_message(bt('Unable to open zip file.'), 'error'); return FALSE; } - bee_copy("$temp/$directory", $destination, FALSE); - bee_delete($temp); + if (file_exists($temp)) { + bee_instant_message('$temp:' . $temp, 'debug'); + } + if (file_exists("$temp/$directory")) { + bee_instant_message('$temp/$directory:' . "$temp/$directory", 'debug'); + } + if ($replace) { + if ($backup) { + $backup_destination = "$_bee_backdrop_root/" . BEE_BACKUP_DIRECTORY; + $backup_destination .= bee_format_date($_SERVER['REQUEST_TIME'], $format = 'YmdHis') . '/'; + $backup_destination .= ($project = 'backdrop') ? 'backdrop/' : 'projects/'; + if (!mkdir($backup_destination, 0660, TRUE )) { + bee_message(bt('Unable to make backup directory.'), 'error'); + return FALSE; + } + $backup_copy = bee_copy($destination, $backup_destination); + if (!$backup_copy) { + bee_message(bt('Unable to make backup copy.'), 'error'); + return FALSE; + } + } + $existing_delete = bee_delete($destination); + if (!$existing_delete) { + bee_message(bt('Unable to delete existing files.'), 'error'); + return FALSE; + } + } + $copy = bee_copy("$temp/$directory", $destination, FALSE); + if (!$copy) { + bee_message(bt('Unable to copy module files to the destination files.'), 'error'); + return FALSE; + } + + $temp_delete = bee_delete($temp); + if (!$temp_delete) { + bee_message(bt('Unable to delete temp files.'), 'warning'); + } return TRUE; } @@ -655,6 +868,47 @@ function download_bee_check_multisite_copy($allow_multisite_copy, $existing_loca } } +/** + * Get the destination path for the project. + * + * @param string $project + * The project system name. + * @param string $type + * The type of project: 'module', 'theme' or 'layout'. + * + * @return string|false + * The absolute destination path. Return FALSE in the event of an error. + */ +function download_bee_get_destination_path($project, $type) { + global $_bee_backdrop_root, $_bee_backdrop_site; + // Add an 's' to the end of the type name. + $type_folder = "${type}s"; + + // Get the directory to download the project into. + if (!empty($_bee_backdrop_site)) { + $destination = "$_bee_backdrop_root/sites/$_bee_backdrop_site/" . $type_folder; + } + elseif (!empty($_bee_backdrop_root)) { + $destination = "$_bee_backdrop_root/" . $type_folder; + } + else { + bee_message(bt("The download destination could not be determined. Re-run the command from within a Backdrop installation, or set the global '--root'/'--site' options."), 'error'); + return FALSE; + } + + // Check if the folder has a "contrib" subfolder within. + if (file_exists("$destination/contrib")) { + $destination .= '/contrib'; + } + + // Finally, add the project name to the end. + $destination .= "/$project"; + if (file_exists($destination)) { + bee_instant_message('$destination:' . $destination, 'debug'); + } + return $destination; +} + /** * Get quota information for the GitHub API anonymous user or token. * @@ -668,74 +922,155 @@ function download_bee_check_multisite_copy($allow_multisite_copy, $existing_loca * @return bool * If there */ -function check_github_api_quota($github_api_token = '', int $minimum_quota = 1) { - // Try to initiate a curl connection to the GitHub Rate Limit API. This does - // not use any quota. - $is_token = FALSE; - if ($curl_handle = curl_init("https://api.github.com/rate_limit")) { +function download_bee_check_github_api_quota($github_api_token = '', int $minimum_quota = 1) { + + $is_token = !empty($github_api_token); + + // Try a call to the GitHub Rate Limit API. This does not use any quota. + $endpoint = 'rate_limit'; + $data = download_bee_github_api_call($endpoint, $github_api_token); + if (!$data) { + bee_message(bt('GitHub API Rate Limit information cannot be retrieved at this time. Please try again later.'), 'error'); + bee_instant_message('GitHub API Rate Limit connection failure', 'log'); + return FALSE; + } + // Return the core quota details. + // var_export($data); + $rate_info = $data['resources']['core']; + $reset_time = bee_format_date($rate_info['reset']); + $used = $rate_info['used']; + $remaining = $rate_info['remaining']; + $quota = $rate_info['limit']; + bee_instant_message("$used/$quota used. Reset at $reset_time", 'debug'); + // Create a debug message with details about the quota. + bee_message(bt('GitHub API Rate Limit: !used/!quota used with !remaining remaining. Quota will reset at !reset_time.', array( + '!used' => $used, + '!remaining' => $remaining, + '!quota' => $quota, + '!reset_time' => $reset_time, + )), 'log'); + // Check the amount remaining. + if ($remaining < $minimum_quota) { + if ($remaining == 0) { + // If there is no quota remaining, provide a meaningful error message. + bee_message(bt('You have reached your rate limit (!quota/hour) for the GitHub API. Your quota will reset at !reset_time.', array( + '!quota' => $quota, + '!reset_time' => $reset_time, + )), 'error'); + } + else { + // If there is some quota remaining, but not enough for the estimated + // number of calls, provide a meaningful error message. + bee_message(bt('You have nearly reached your rate limit (!quota/hour) for the GitHub API and it is estimated that the amount remaining will not be enough to complete this operation. Your quota will reset at !reset_time.', array( + '!quota' => $quota, + '!reset_time' => $reset_time, + )), 'error'); + } + // If there is not a token in use, then also provide an information + // message about using a token. + if (!$is_token) { + bee_message(bt('The GitHub API rate limit is 60/hour for anonymous use. You can increase this by using a token. See the help or wiki for more information.'), 'info'); + } + return FALSE; + } + else { + // There is still enough quota remaining. + return TRUE; + } +} + +/** + * Call the Github API. + * + * @param string $endpoint + * The GitHub API endpoint to use. + * @param string $github_api_token + * The GitHub Personal Access Token to check. Anonymous if left blank. + * @param string $request_type + * Set the request type to be either 'json' (default) or 'raw'. + * + * @return array|false + * Returns an array of the response if successful; FALSE otherwise. + */ +function download_bee_github_api_call($endpoint, $github_api_token = '', $request_type = 'json') { + + if ($curl_handle = curl_init(BEE_GITHUB_API_URL . "$endpoint")) { // Prepare headers for curl request. $curlopt_httpheader = array(); - $curlopt_httpheader[] = 'Content-Type: application/json'; + switch ($request_type) { + case 'raw': + $curlopt_httpheader[] = 'Accept: application/vnd.github.v3.raw'; + break; + default: + $curlopt_httpheader[] = 'Content-Type: application/json'; + break; + } + // If a token has been passed, add this to the header. if (!empty($github_api_token)) { $curlopt_httpheader[] = "Authorization: token $github_api_token"; - $is_token = TRUE; } + + // Compile the options for the curl request. curl_setopt($curl_handle, CURLOPT_HTTPHEADER, $curlopt_httpheader); - // Set the other curl options. curl_setopt($curl_handle, CURLOPT_USERAGENT, BEE_USERAGENT); curl_setopt($curl_handle, CURLOPT_HEADER, 0); curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); - // Get the rate information. - $data = curl_exec($curl_handle); + + // Execute and close the curl request. + $response = curl_exec($curl_handle); curl_close($curl_handle); - $data = json_decode($data, TRUE); - if (empty($data)) { - bee_message(bt('GitHub API Rate Limit information cannot be retrieved at this time. Please try again later.'), 'error'); - return FALSE; - } - // Return the core quota details. - $rate_info = $data['resources']['core']; - $reset_time = date('H:m T', $rate_info['reset']); - $used = $rate_info['used']; - $remaining = $rate_info['remaining']; - $quota = $rate_info['limit']; - // Create a debug message with details about the quota. - bee_message(bt('GitHub API Rate Limit: !used/!quota used with !remaining remaining. Quota will reset at !reset_time.', array( - '!used' => $used, - '!remaining' => $remaining, - '!quota' => $quota, - '!reset_time' => $reset_time, - )), 'log'); - // Check the amount remaining. - if ($remaining < $minimum_quota) { - if ($remaining == 0) { - // If there is no quota remaining, provide a meaningful error message. - bee_message(bt('You have reached your rate limit (!quota/hour) for the GitHub API. Your quota will reset at !reset_time.', array( - '!quota' => $quota, - '!reset_time' => $reset_time, - )), 'error'); - } - else { - // If there is some quota remaining, but not enough for the estimated - // number of calls, provide a meaningful error message. - bee_message(bt('You have nearly reached your rate limit (!quota/hour) for the GitHub API and it is estimated that the amount remaining will not be enough to complete this operation. Your quota will reset at !reset_time.', array( - '!quota' => $quota, - '!reset_time' => $reset_time, - )), 'error'); - } - // If there is not a token in use, then also provide an information - // message about using a token. - if (!$is_token) { - bee_message(bt('The GitHub API rate limit is 60/hour for anonymous use. You can increase this by using a token. See the help or wiki for more information.'), 'info'); - } - return FALSE; - } - else { - // There is still enough quota remaining. - return TRUE; + + // Process the output according to the request type. + switch ($request_type) { + case 'raw': + // $output = explode("\n", $response); + $output = array($response); + break; + default: + $output = json_decode($response, TRUE); + break; } + return $output; + } + else { + return FALSE; } - // If we can't initiate the curl connection then return FALSE. - return FALSE; +} + +/** + * Check whether a URL exists. + * + * This helper function provides a simple check for URLs that does require any + * use of the GitHub API call quota. + * + * @param string $url + * The URL to check. + * + * @return bool + * Returns TRUE if a valid result is returned. FALSE otherwise. + */ +function download_bee_check_url_exists($url) { + // Initiate the curl request. + $curl_handle = curl_init($url); + + // Compile the options for the curl request. + // Set the user agent string. + curl_setopt($curl_handle, CURLOPT_USERAGENT, BEE_USERAGENT); + // Allow curl to follow redirects. + curl_setopt($curl_handle, CURLOPT_FOLLOWLOCATION, TRUE); + // Specify that we don't need the body of the response. + curl_setopt($curl_handle, CURLOPT_NOBODY, TRUE); + + // Execute the curl request. + curl_exec($curl_handle); + + // Get the response code. + $return_code = curl_getinfo($curl_handle, CURLINFO_RESPONSE_CODE); + + // Close the curl request. + curl_close($curl_handle); + + // Return the result. + return $return_code == 200; } diff --git a/includes/globals.inc b/includes/globals.inc index 343d309d..223b2af0 100644 --- a/includes/globals.inc +++ b/includes/globals.inc @@ -97,6 +97,11 @@ define('BEE_BOOTSTRAP_LANGUAGE', 7); // 9th phase: Backdrop is fully loaded; validate and fix input data. define('BEE_BOOTSTRAP_FULL', 8); +/** + * Define constant for path to backup directory. + */ +define('BEE_BACKUP_DIRECTORY', '../bee-backup/'); + /** * Define constant for Bee's current version. * From 372537c47b08bebef1c0fc6ebddd588a040f992f Mon Sep 17 00:00:00 2001 From: Martin Price Date: Tue, 2 Jul 2024 21:05:31 +0100 Subject: [PATCH 02/15] Further download changes to date --- bee.php | 1 + commands/download.bee.inc | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/bee.php b/bee.php index fc8326d3..821d5b7b 100755 --- a/bee.php +++ b/bee.php @@ -22,6 +22,7 @@ require_once __DIR__ . '/includes/filesystem.inc'; require_once __DIR__ . '/includes/input.inc'; require_once __DIR__ . '/includes/globals.inc'; +require_once __DIR__ . '/includes/info.inc'; // Main execution code. bee_initialize_server(); diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 8859acd4..99799d71 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -14,13 +14,6 @@ define('BEE_USERAGENT', 'Bee - the command line tool for Backdrop CMS'); */ define('BEE_GITHUB_API_URL', 'https://api.github.com/'); -/** - * Include the common functions file so we can re-use some Backdrop functions - * for handling .info files and dependencies. - */ -global $_bee_backdrop_root; -require_once $_bee_backdrop_root . '/core/includes/common.inc'; - /** * Implements hook_bee_command(). */ @@ -35,9 +28,16 @@ function download_bee_command() { ), 'multiple_argument' => 'projects', 'options' => array( - 'hide-progress' => array( - 'description' => bt('Deprecated, will get removed in a future version.'), - 'short' => 'h', + 'release' => array( + 'description' => bt("Specify a release tag or one of the following keywords:") + . "\n - " . bt("'dev' (download the dev version from the default branch.)") + . "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified using the 'branch' option.)") + . "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"), + 'value' => bt("The release tag or keyword."), + ), + 'branch' => array( + 'description' => bt("If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise."), + 'value' => bt('The branch name.'), ), 'allow-multisite-copy' => array( 'description' => bt('Override the check that would prevent the project being downloaded to a multisite site if the project exists in the shared project directory.'), @@ -64,9 +64,11 @@ function download_bee_command() { ), 'optional_arguments' => array('directory'), 'options' => array( - 'hide-progress' => array( - 'description' => bt('Deprecated, will get removed in a future version.'), - 'short' => 'h', + 'release' => array( + 'description' => bt("Specify a release tag or one of the following keywords:") + . "\n - " . bt("'dev' (download the dev version from the default branch.)") + . "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"), + 'value' => bt("The release tag or keyword."), ), 'github-token' => array( 'description' => bt('A Github Personal Access Token (Classic) that can be used to extend the GitHub API rate.'), @@ -623,12 +625,12 @@ function download_bee_get_project_info($project, $organization, $release = '', $ bee_instant_message('$endpoint:' . $endpoint, 'debug'); // // Retrieve the info file as an array of lines. $info_file = download_bee_github_api_call($endpoint, $github_api_token, 'raw'); - $info_parsed = backdrop_parse_info_format($info_file[0]); + $info_parsed = bee_parse_info_format($info_file[0]); bee_instant_message(bt('The info file data:'), 'debug', $info_parsed); $info['type'] = $info_parsed['type']; if (isset($info_parsed['dependencies'])) { foreach ($info_parsed['dependencies'] as $dependency) { - $dependency = backdrop_parse_dependency($dependency); + $dependency = bee_parse_dependency($dependency); $info['dependencies'][] = $dependency; } } From 5e4d5558f9a8535c8703e8c3eb6ea934d4d597aa Mon Sep 17 00:00:00 2001 From: Martin Price Date: Tue, 2 Jul 2024 23:59:07 +0100 Subject: [PATCH 03/15] Add info.inc with copies of Backdrops info file handling functions --- includes/info.inc | 159 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 includes/info.inc diff --git a/includes/info.inc b/includes/info.inc new file mode 100644 index 00000000..228558e0 --- /dev/null +++ b/includes/info.inc @@ -0,0 +1,159 @@ + 1) { + $info_sections = array(); + for ($n = 0; $n < $split_count; $n = $n + 2) { + $info_sections[$split[$n]] = bee_parse_info_format($split[$n + 1], FALSE); + } + return $info_sections; + } + } + + if (preg_match_all(' + @^\s* # Start at the beginning of a line, ignoring leading whitespace + ((?: + [^=;\[\]]| # Key names cannot contain equal signs, semi-colons or square brackets, + \[[^\[\]]*\] # unless they are balanced and not nested + )+?) + \s*=\s* # Key/value pairs are separated by equal signs (ignoring white-space) + (?: + ("(?:[^"]|(?<=\\\\)")*")| # Double-quoted string, which may contain slash-escaped quotes/slashes + (\'(?:[^\']|(?<=\\\\)\')*\')| # Single-quoted string, which may contain slash-escaped quotes/slashes + ([^\r\n]*?) # Non-quoted string + )\s*$ # Stop at the next end of a line, ignoring trailing whitespace + @msx', $data, $matches, PREG_SET_ORDER)) { + foreach ($matches as $match) { + // Fetch the key and value string. + $i = 0; + $parts = array(); + foreach (array('key', 'value1', 'value2', 'value3') as $var) { + $parts[$var] = isset($match[++$i]) ? $match[$i] : ''; + } + $value = stripslashes(substr($parts['value1'], 1, -1)) . stripslashes(substr($parts['value2'], 1, -1)) . $parts['value3']; + + // Parse array syntax. + $keys = preg_split('/\]?\[/', rtrim($parts['key'], ']')); + $last = array_pop($keys); + $parent = &$info; + + // Create nested arrays. + foreach ($keys as $key) { + if ($key == '') { + $key = count($parent); + } + if (!isset($parent[$key]) || !is_array($parent[$key])) { + $parent[$key] = array(); + } + $parent = &$parent[$key]; + } + + // Handle PHP constants. + if (preg_match('/^\w+$/', $value) && defined($value)) { + $value = constant($value); + } + + // Insert actual value. + if ($last == '') { + $last = count($parent); + } + $parent[$last] = $value; + } + } + + return $info; +} + +/** + * @see backdrop_parse_dependency + */ +function bee_parse_dependency($dependency) { + $value = array(); + // Split out the optional project name. + if (strpos($dependency, ':')) { + list($project_name, $dependency) = explode(':', $dependency); + $value['project'] = $project_name; + } + // We use named subpatterns and support every op that version_compare + // supports. Also, op is optional and defaults to equals. + $p_op = '(?P!=|==|=|<|<=|>|>=|<>)?'; + // Core version is always optional: 1.x-2.x and 2.x is treated the same. + $p_core = '(?:' . preg_quote(BACKDROP_CORE_COMPATIBILITY) . '-)?'; + $p_major = '(?P\d+)'; + // By setting the minor version to x, branches can be matched. + $p_minor = '(?P(?:\d+|x)(?:-[A-Za-z]+\d+)?)'; + $p_patch = '(?P(?:\d+|x)(?:-[A-Za-z]+\d+)?)?'; + $parts = explode('(', $dependency, 2); + $value['name'] = trim($parts[0]); + if (isset($parts[1])) { + $value['original_version'] = '(' . $parts[1]; + foreach (explode(',', $parts[1]) as $version) { + if (preg_match("/^\s*$p_op\s*$p_core$p_major\.$p_minor\.?$p_patch/", $version, $matches)) { + $op = !empty($matches['operation']) ? $matches['operation'] : '='; + if ($matches['minor'] == 'x') { + // Backdrop considers "2.x" to mean any version that begins with + // "2" (e.g. 2.0, 2.9 are all "2.x"). PHP's version_compare(), + // on the other hand, treats "x" as a string; so to + // version_compare(), "2.x" is considered less than 2.0. This + // means that >=2.x and <2.x are handled by version_compare() + // as we need, but > and <= are not. + if ($op == '>' || $op == '<=') { + $matches['major']++; + } + // Equivalence can be checked by adding two restrictions. + if ($op == '=' || $op == '==') { + $value['versions'][] = array( + 'op' => '<', + 'version' => ($matches['major'] + 1) . '.x', + ); + $op = '>='; + } + } + + if (isset($matches['patch']) && ($matches['patch'] === '0' || $matches['patch'])) { + if ($matches['patch'] == 'x' && $matches['minor'] !== 'x') { + // See comments above about "x" in minor. + // Same principle applies to patch in relation to minor. + if ($op == '>' || $op == '<=') { + $matches['minor']++; + } + if ($op == '=' || $op == '==') { + $value['versions'][] = array( + 'op' => '<', + 'version' => $matches['major'] . '.' . ($matches['minor'] + 1) . '.x', + ); + $op = '>='; + } + } + } + $version = $matches['major'] . '.' . $matches['minor']; + $version .= (isset($matches['patch']) && ($matches['patch'] === '0' || $matches['patch'])) ? '.' . $matches['patch'] : ''; + $value['versions'][] = array('op' => $op, 'version' => $version); + } + } + } + return $value; +} From cf88998c62c98064cb4be93c8ef41f415bac05a1 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Mon, 15 Jul 2024 23:20:30 +0100 Subject: [PATCH 04/15] Merge 1.x-1.x --- commands/download.bee.inc | 2 -- 1 file changed, 2 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 9bc20c85..d3ffdeea 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -4,8 +4,6 @@ * Command(s) for downloading Backdrop projects. */ - - /** * The API URL used for GitHub API calls. */ From b1a9e041fd67fbc3e52b284555759e1b3d750bdc Mon Sep 17 00:00:00 2001 From: Martin Price Date: Wed, 24 Jul 2024 09:33:43 +0100 Subject: [PATCH 05/15] progress to date --- commands/download.bee.inc | 267 +++++++++++++++++++++++--------------- 1 file changed, 164 insertions(+), 103 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index d3ffdeea..4d09aa57 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -13,27 +13,24 @@ define('BEE_GITHUB_API_URL', 'https://api.github.com/'); * Implements hook_bee_command(). */ function download_bee_command() { + $projects_description = bt('One or more contrib projects to download. You can specify a release tag or branch using the format:'); + $projects_description .= "\n " . bt('project:release_tag/keyword[:branch]'); + $projects_description .= "\n\n " . bt("Specify a release tag or one of the following keywords:"); + $projects_description .= "\n - " . bt("'dev' (download the dev version from the default branch.)"); + $projects_description .= "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified using the 'branch' option.)"); + $projects_description .= "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"); + $projects_description .= "\n\n " . bt("If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise."); + return array( 'download' => array( 'description' => bt('Download Backdrop contrib projects.'), 'callback' => 'download_bee_callback', 'group' => 'projects', 'arguments' => array( - 'projects' => bt('One or more contrib projects to download.'), + 'projects' => $projects_description, ), 'multiple_argument' => 'projects', 'options' => array( - 'release' => array( - 'description' => bt("Specify a release tag or one of the following keywords:") - . "\n - " . bt("'dev' (download the dev version from the default branch.)") - . "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified using the 'branch' option.)") - . "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"), - 'value' => bt("The release tag or keyword."), - ), - 'branch' => array( - 'description' => bt("If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise."), - 'value' => bt('The branch name.'), - ), 'allow-multisite-copy' => array( 'description' => bt('Override the check that would prevent the project being downloaded to a multisite site if the project exists in the shared project directory.'), 'short' => 'f', @@ -128,23 +125,49 @@ function download_bee_callback($arguments, $options) { // Iterate through the list of projects manually so any dependencies can be // added to the list of projects to download. $project_count = 0; - while ($project_count < count($arguments['projects'])) { - $project = $arguments['projects'][$project_count]; + $projects = $arguments['projects']; + while ($project_count < count($projects)) { + $project = $projects[$project_count]; + $project_details = explode(':', $project); + $detail_count = count($project_details); + switch ($detail_count) { + case 1: + $project = array( + 'name' => $project_details[0], + 'release' => '', + 'branch' => '', + ); + break; + case 2: + $project = array( + 'name' => $project_details[0], + 'release' => $project_details[1], + 'branch' => '', + ); + break; + case 3: + $project = array( + 'name' => $project_details[0], + 'release' => $project_details[1], + ); + $project['branch'] = ($project['release'] == 'branch') ? $project_details[2] : ''; + } + // Check if the project exists by trying to get the repo homepage. $organization = 'backdrop-contrib'; - $url = "https://github.com/$organization/$project"; + $url = "https://github.com/$organization/" . $project['name']; if (!download_bee_check_url_exists($url)) { bee_message(bt("The '!project' project repository could not be found. Please check your spelling and try again.", array( - '!project' => $project, + '!project' => $project['name'], )), 'error'); } else { - $info = download_bee_get_project_info($project, $organization, '', '', $github_api_token); + $info = download_bee_get_project_info($project['name'], $organization, $project['release'], $project['branch'], $github_api_token); if (empty($info)) { // If getting the info file has failed, show an error message here and // exit. bee_message(bt("The info file for !project cannot be retrieved at this time.", array( - '!project' => $project, + '!project' => $project['name'], )), 'error'); return; } @@ -153,10 +176,7 @@ function download_bee_callback($arguments, $options) { $dependencies = $info['dependencies']; foreach ($dependencies as $dependency_detail) { $submodule = ''; - // Remove any minimum version requirements to get just the project - // name. - // $dependency = explode(" ", $dependency, 2); - // $dependency = $dependency[0]; + // Check whether the dependency is a submodule. if (isset($dependency_detail['project'])) { // If so, get the submodule. @@ -181,7 +201,7 @@ function download_bee_callback($arguments, $options) { // If getting the info file has failed, show an error message here // and exit. bee_message(bt("!dependency is a dependency of !project but the info file for !dependency cannot be retrieved at this time.", array( - '!project' => $project, + '!project' => $project['name'], '!dependency' => $dependency, )), 'error'); continue; @@ -193,24 +213,24 @@ function download_bee_callback($arguments, $options) { // If project does not exist in the site file system, or it is // but a multisite copy is allowed, then check whether it is // already in the download list. - if (in_array($dependency, $arguments['projects'])) { + if (in_array($dependency, $projects)) { // If project is already in the list to download, do not add to // the list to download again. bee_message(bt("The '!dependency' !dependency_type is also required by the '!project' !project_type.", array( '!dependency' => !empty($submodule) ? "$dependency:$submodule" : $dependency, '!dependency_type' => (!empty($submodule) ? 'sub' : '') . $dependency_info['type'], - '!project' => $project, + '!project' => $project['name'], '!project_type' => $info['type'], )), 'status'); } else { // If project is not already in download list, then add to list // and prepare message. - $arguments['projects'][] = $dependency; + $projects[] = $dependency; bee_message(bt("The '!dependency' !dependency_type will also be downloaded, as it is required by the '!project' !project_type.", array( '!dependency' => !empty($submodule) ? "$dependency:$submodule" : $dependency, '!dependency_type' => (!empty($submodule) ? 'sub' : '') . $dependency_info['type'], - '!project' => $project, + '!project' => $project['name'], '!project_type' => $info['type'], )), 'status'); } @@ -222,7 +242,7 @@ function download_bee_callback($arguments, $options) { bee_message(bt("The '!dependency' !dependency_type is required by the '!project' !project_type but already exists at '!dependency_location'.", array( '!dependency' => $dependency, '!dependency_type' => $dependency_info['type'], - '!project' => $project, + '!project' => $project['name'], '!project_type' => $info['type'], '!dependency_location' => $dependency_existing_location, )), 'status'); @@ -234,19 +254,19 @@ function download_bee_callback($arguments, $options) { // Get the project type. if (empty($info['type'])) { bee_message(bt("The 'type' of project '!project' could not be determined.", array( - '!project' => $project, + '!project' => $project['name'], )), 'error'); return; } - $destination = download_bee_get_destination_path($project, $info['type']); + $destination = download_bee_get_destination_path($project['name'], $info['type']); bee_instant_message('$destination:' . $destination, 'debug'); // Check if the project exists within the site file system. - $project_existing_location = download_bee_check_project_exists($project, $info['type']); + $project_existing_location = download_bee_check_project_exists($project['name'], $info['type']); $project_allow_multisite_copy = download_bee_check_multisite_copy($allow_multisite_copy, $project_existing_location); if ($project_existing_location != FALSE && !$project_allow_multisite_copy) { bee_message(bt("'!project' already exists in '!existing_location'.", array( - '!project' => $project, + '!project' => $project['name'], '!existing_location' => $project_existing_location, )), 'error'); return; @@ -259,9 +279,11 @@ function download_bee_callback($arguments, $options) { } // Download the project. - if (download_bee_download_project($project, $info['download_url'], $destination, $info['branch'])) { - bee_message(bt("'!project' was downloaded into '!directory'.", array( - '!project' => $project, + if (download_bee_download_project($project['name'], $info['download_url'], $destination, $info['branch'])) { + bee_message(bt("'!project' (!release) was downloaded into '!directory'. For more information see !url", array( + '!project' => $project['name'], + '!release' => $info['release']['tag_name'], + '!url' => $info['release']['release_url'], '!directory' => $destination, )), 'success'); } @@ -434,68 +456,17 @@ function download_bee_get_project_info($project, $organization, $release = '', $ } case 'select': $preferred_option = 'select'; - // Create the array for options and hardcode the 'dev' option. - $select_options = array( - 'dev' => bt("Dev - Download the current default branch."), - ); - // Check GitHub API quota to ensure we can continue. - if (!download_bee_check_github_api_quota($github_api_token)) { - return FALSE; - } - // Get the list of releases. - $endpoint = "$endpoint_base/releases"; - $releases = download_bee_github_api_call($endpoint, $github_api_token); - // If one or more releases exist, then loop through them to get more - // information. - if (!empty($releases)) { - foreach ($releases as $release_data) { - // Convert the array to being an associative array keyed by tag_name. - $releases_by_tag[$release_data['tag_name']] = $release_data; - // If the release is not a prerelease, also put into an array to check - // for latest release. - if (!$release_data['prerelease']) { - $full_release_list[] = $release_data['tag_name']; - } - } - // If full releases exist, find the latest release. - if (!empty($full_release_list)) { - usort($full_release_list, 'version_compare'); - $latest_release_tag = end($full_release_list); - $default = $latest_release_tag; - } - else { - $latest_release_tag = NULL; - $default = 'dev'; - } - // Add each release to the list of options. - foreach ($releases_by_tag as $key => $release_data) { - // Define any suffixes needed for 'latest' or 'pre-release'. - $suffix = ''; - if ($key == $latest_release_tag) { - $suffix = bt(" (latest)"); - } - if ($release_data['prerelease']) { - $suffix = bt(" (pre-release)"); - } - $select_options[$key] = $key . $suffix; - } - // Set the message to be used. - $message = bt("Select the version you wish to download."); - } - else { - // If no releases exist, tailor the message and default. - $message = bt("No releases exist but you can download the dev version."); - $default = 'dev'; - } - // Present the choice to the user. - $selection = bee_choice($select_options, $message, $default); + + $selection = download_bee_select_version($project, $organization, $github_api_token); + // Use the selection to set variables that will be used in the next step. - if ($selection == 'dev') { + if ($selection['value'] == 'dev') { $final_option = 'dev'; } else { $final_option = 'defined_release'; - $defined_release = $selection; + $defined_release = $selection['value']; + $release_data = $selection['data']; } break; default: @@ -508,19 +479,32 @@ function download_bee_get_project_info($project, $organization, $release = '', $ // If the release page exists, then this entry is valid and we can set // the final option. $final_option = 'defined_release'; + $endpoint = "$endpoint_base/releases/tags/$defined_release"; + $release_data = download_bee_github_api_call($endpoint, $github_api_token); } else { // If the release page does not exist offer to get the dev version // instead. - $choice = bee_confirm(bt("The release you have specified (!release_tag) for '!organization/!project' does not exist. Do you want to download the dev version instead?", array( + $choice = bee_confirm(bt("The release you have specified (!release_tag) for '!organization/!project' does not exist. Do you want to select a version instead?", array( '!release_tag' => $defined_release, '!organization' => $organization, '!project' => $project, )), TRUE); if ($choice) { // If 'Yes' mode was enabled or user responded 'Yes', then continue - // with dev version. - $final_option = 'dev'; + // with selection. + $selection = download_bee_select_version($project, $organization, $github_api_token); + + // Use the selection to set variables that will be used in the next + // step. + if ($selection['value'] == 'dev') { + $final_option = 'dev'; + } + else { + $final_option = 'defined_release'; + $defined_release = $selection['value']; + $release_data = $selection['data']; + } } else { // If 'Yes' mode was not enabled and user responded 'No', then provide @@ -542,17 +526,17 @@ function download_bee_get_project_info($project, $organization, $release = '', $ } // Get the latest release information. $endpoint = "$endpoint_base/releases/latest"; - $latest_release = download_bee_github_api_call($endpoint, $github_api_token); - $defined_release = $latest_release['tag_name']; + $release_data = download_bee_github_api_call($endpoint, $github_api_token); + $defined_release = $release_data['tag_name']; bee_instant_message('$defined_release:' . $defined_release, 'debug'); - $defined_release_published_date = $latest_release['published_at']; - $defined_release_url = $latest_release['html_url']; + // $defined_release_published_date = $latest_release['published_at']; + // $defined_release_url = $latest_release['html_url']; $download_url = "https://github.com/$organization/$project/releases/download/$defined_release/$project.zip"; bee_instant_message('$download_url:' . $download_url, 'debug'); $release_detail = array( 'tag_name' => $defined_release, - 'published_at' => $defined_release_published_date, - 'release_url' => $defined_release_url, + 'published_at' => $release_data['published_at'], + 'release_url' => $release_data['html_url'], ); break; case 'dev': @@ -584,8 +568,8 @@ function download_bee_get_project_info($project, $organization, $release = '', $ $download_url = "https://github.com/$organization/$project/releases/download/$defined_release/$project.zip"; $release_detail = array( 'tag_name' => $defined_release, - 'published_at' => $defined_release_published_date, - 'release_url' => $defined_release_url, + 'published_at' => $release_data['published_at'], + 'release_url' => $release_data['html_url'], ); break; } @@ -634,6 +618,83 @@ function download_bee_get_project_info($project, $organization, $release = '', $ return $info; } +/** + * Helper function to select version. + */ +function download_bee_select_version($project, $organization, $github_api_token = '') { + // Create the array for options and hardcode the 'dev' option. + $select_options = array( + 'dev' => bt("Dev - Download the current default branch."), + ); + // Check GitHub API quota to ensure we can continue. + if (!download_bee_check_github_api_quota($github_api_token)) { + return FALSE; + } + // Get the list of releases. + $endpoint = "repos/$organization/$project/releases"; + $releases = download_bee_github_api_call($endpoint, $github_api_token); + // If one or more releases exist, then loop through them to get more + // information. + if (!empty($releases)) { + foreach ($releases as $release_data) { + // Convert the array to being an associative array keyed by tag_name. + $releases_by_tag[$release_data['tag_name']] = $release_data; + // If the release is not a prerelease, also put into an array to check + // for latest release. + if (!$release_data['prerelease']) { + $full_release_list[] = $release_data['tag_name']; + } + } + // If full releases exist, find the latest release. + if (!empty($full_release_list)) { + usort($full_release_list, 'version_compare'); + $latest_release_tag = end($full_release_list); + $default = $latest_release_tag; + } + else { + $latest_release_tag = NULL; + $default = 'dev'; + } + // Add each release to the list of options. + foreach ($releases_by_tag as $key => $release_data) { + // Define any suffixes needed for 'latest' or 'pre-release'. + $suffix = ''; + if ($key == $latest_release_tag) { + $suffix = bt(" (latest)"); + } + if ($release_data['prerelease']) { + $suffix = bt(" (pre-release)"); + } + $select_options[$key] = $key . $suffix; + } + // Set the message to be used. + $message = bt("Select the version you wish to download."); + } + else { + // If no releases exist, tailor the message and default. + $message = bt("No releases exist but you can download the dev version."); + $default = 'dev'; + } + // Present the choice to the user. + $selection = bee_choice($select_options, $message, $default); + switch ($selection) { + case 'dev': + $result = array( + 'value' => 'dev', + ); + break; + default: + $result = array( + 'value' => $selection, + 'data' => $releases_by_tag[$selection], + ); + } + return $result; +} + + + + /** * Download a project. * From d6d285b2217aed949d5226506b0e62072848256e Mon Sep 17 00:00:00 2001 From: Martin Price Date: Wed, 2 Oct 2024 19:23:44 +0100 Subject: [PATCH 06/15] Download command enhancements --- CHANGELOG.md | 8 ++- commands/download.bee.inc | 145 ++++++++++++++++++++++++++++++-------- docs/Usage.md | 40 ++++++++--- 3 files changed, 152 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 688ba5d7..77f5156e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,19 @@ and this project follows the which is based on the major version of Backdrop CMS with a semantic version system for each contributed module, theme and layout. -## [Unreleased] - 2024-09-25 +## [Unreleased] - 2024-TBC ### Added - An option for the `db-import` command to allow import from newer MariaDB servers with the enable sandbox command in the dump file if the destination database or client does not support it. - Tooling for the lando recipe to support Xdebug with VS Code. +- The ability to download specified releases or branches of modules, themes, +layouts or Backdrop itself. + +### Changed +- The functions within the download command have been made more flexible and +better able to support the coming 'update' command. ### Fixed - Unhandled errors and warnings if commands run outside Backdrop root and/or diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 1f887fba..05671591 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -13,13 +13,14 @@ define('BEE_GITHUB_API_URL', 'https://api.github.com/'); * Implements hook_bee_command(). */ function download_bee_command() { - $projects_description = bt('One or more contrib projects to download. You can specify a release tag or branch using the format:'); + $projects_description = bt('One or more contrib projects to download. You can specify a release tag or branch for each project using the format:'); $projects_description .= "\n " . bt('project:release_tag/keyword[:branch]'); $projects_description .= "\n\n " . bt("Specify a release tag or one of the following keywords:"); $projects_description .= "\n - " . bt("'dev' (download the dev version from the default branch.)"); - $projects_description .= "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified using the 'branch' option.)"); + $projects_description .= "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified with "); $projects_description .= "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"); $projects_description .= "\n\n " . bt("If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise."); + $projects_description .= "\n\n " . bt("By default, 'latest' is selected if no release is specified. Dependencies will all download the 'latest' so if you do want a different version, download these first."); return array( 'download' => array( @@ -46,9 +47,13 @@ function download_bee_command() { ), 'aliases' => array('dl', 'pm-download'), 'examples' => array( - 'bee download webform' => bt('Download the Webform module.'), + 'bee download webform' => bt('Download the latest version of the Webform module.'), 'bee download simplify thesis bamboo' => bt('Download the Simplify module, Thesis theme, and Bamboo layout.'), 'bee --site=site_name download simplify --allow-multisite-copy' => bt('Download an additional copy of the Simplify module into the site_name multisite module folder.'), + 'bee download simplify:dev' => bt('Download the dev version of the Simplify module.'), + 'bee download paragraphs:branch:1.x-1.2' => bt('Download the 1.x-1.2 branch of the Paragraphs module'), + 'bee download simplify:1.x-1.2.4' => bt('Download the 1.x-1.2.4 release of the Simplify module'), + 'bee download simplify:select' => bt('Select from a list of releases and the dev version for the Simplify module.'), ), ), 'download-core' => array( @@ -64,9 +69,10 @@ function download_bee_command() { ), 'optional_arguments' => array('directory'), 'options' => array( - 'release' => array( + 'version' => array( 'description' => bt("Specify a release tag or one of the following keywords:") . "\n - " . bt("'dev' (download the dev version from the default branch.)") + . "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified by appending ':branch_name')") . "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"), 'value' => bt("The release tag or keyword."), ), @@ -78,6 +84,10 @@ function download_bee_command() { 'aliases' => array('dl-core'), 'examples' => array( 'bee download-core ../backdrop' => bt("Download Backdrop into a 'backdrop' directory in the parent folder."), + 'bee download-core --version=1.28.3' => bt("Download the 1.28.3 release of Backdrop into the current directory."), + 'bee download-core --version=dev' => bt("Download the dev version of Backdrop into the current directory."), + 'bee download-core --version=branch:1.29.x' => bt("Download the 1.29.x branch of Backdrop into the current directory."), + 'bee download-core --version=select' => bt("Select from a list of releases of Backdrop to download into the current directory."), ), ), ); @@ -137,6 +147,7 @@ function download_bee_callback($arguments, $options) { while ($project_count < count($projects)) { $project = $projects[$project_count]; $project_details = explode(':', $project); + bee_instant_message('$project_details: ', 'debug', $project_details); $detail_count = count($project_details); switch ($detail_count) { case 1: @@ -288,12 +299,28 @@ function download_bee_callback($arguments, $options) { // Download the project. if (download_bee_download_project($project['name'], $info['download_url'], $destination, $info['branch'])) { - bee_message(bt("'!project' (!release) was downloaded into '!directory'. For more information see !url", array( - '!project' => $project['name'], - '!release' => $info['release']['tag_name'], - '!url' => $info['release']['release_url'], - '!directory' => $destination, - )), 'success'); + // If the download is successful then display an appropriate message. + // Check if a release. + if (isset($info['release'])) { + // If download is a release, whether latest or defined, show the + // release tag and link to release notes. + bee_message(bt("'!project' (!release) was downloaded into '!directory'. For more information see !url", array( + '!project' => $project['name'], + '!release' => $info['release']['tag_name'], + '!url' => $info['release']['release_url'], + '!directory' => $destination, + )), 'success'); + } + else { + // If download is not a release it must be a branch so we should show + // the branch name. + bee_message(bt("'!project' (!branch) was downloaded into '!directory'.", array( + '!project' => $project['name'], + '!branch' => $info['branch'], + '!directory' => $destination, + )), 'success'); + } + } } $project_count++; @@ -307,6 +334,34 @@ function download_core_bee_callback($arguments, $options) { // Set the GitHub Token if entered. $github_api_token = $options['github-token'] ?? ''; + // Get the version details if entered. + $version = $options['version'] ?? ''; + + if (!empty($version)) { + $version_details = explode(':', $version); + bee_instant_message('$version_details: ', 'debug', $version_details); + $detail_count = count($version_details); + switch ($detail_count) { + case 1: + $core_version = array( + 'release' => $version_details[0], + 'branch' => '', + ); + break; + case 2: + $core_version = array( + 'release' => $version_details[0], + ); + $core_version['branch'] = ($core_version['release'] == 'branch') ? $version_details[1] : ''; + } + } + else { + // If version is not specified then set $core_version accordingly. + $core_version = array( + 'release' => '', + 'branch' => '', + ); + } // Estimate the number of API calls. Only 1 is needed for Backdrop Core. $api_call_estimate = 1; @@ -315,7 +370,7 @@ function download_core_bee_callback($arguments, $options) { return; } - $info = download_bee_get_project_info('backdrop', 'backdrop', '', '', $github_api_token); + $info = download_bee_get_project_info('backdrop', 'backdrop', $core_version['release'], $core_version['branch'], $github_api_token); // Get or create the directory to download Backdrop into. $destination = !empty($arguments['directory']) ? $arguments['directory'] : getcwd(); @@ -337,9 +392,25 @@ function download_core_bee_callback($arguments, $options) { // Download Backdrop. if (download_bee_download_project('backdrop', $info['download_url'], $destination, $info['branch'])) { - bee_message(bt("Backdrop was downloaded into '!directory'.", array( - '!directory' => $destination, - )), 'success'); + // If the download is successful then display an appropriate message. + // Check if a release. + if (isset($info['release'])) { + // If download is a release, whether latest or defined, show the + // release tag and link to release notes. + bee_message(bt("Backdrop (!release) was downloaded into '!directory'. For more information see !url", array( + '!release' => $info['release']['tag_name'], + '!url' => $info['release']['release_url'], + '!directory' => $destination, + )), 'success'); + } + else { + // If download is not a release it must be a branch so we should show + // the branch name. + bee_message(bt("Backdrop (!branch) was downloaded into '!directory'.", array( + '!branch' => $info['branch'], + '!directory' => $destination, + )), 'success'); + } } } @@ -462,6 +533,7 @@ function download_bee_get_project_info($project, $organization, $release = '', $ return FALSE; } } + break; case 'select': $preferred_option = 'select'; @@ -592,6 +664,8 @@ function download_bee_get_project_info($project, $organization, $release = '', $ // populate the info array with conditional elements. switch ($final_option) { case 'dev': + $ref = $branch; + $info['branch'] = $branch; case 'branch': $ref = $branch; $info['branch'] = $branch; @@ -607,18 +681,26 @@ function download_bee_get_project_info($project, $organization, $release = '', $ if (!download_bee_check_github_api_quota($github_api_token)) { return FALSE; } - // Compile the endpoint. - $endpoint = "$endpoint_base/contents/$project.info?ref=$ref"; - bee_instant_message('$endpoint:' . $endpoint, 'debug'); - // // Retrieve the info file as an array of lines. - $info_file = download_bee_github_api_call($endpoint, $github_api_token, 'raw'); - $info_parsed = bee_parse_info_format($info_file[0]); - bee_instant_message(bt('The info file data:'), 'debug', $info_parsed); - $info['type'] = $info_parsed['type']; - if (isset($info_parsed['dependencies'])) { - foreach ($info_parsed['dependencies'] as $dependency) { - $dependency = bee_parse_dependency($dependency); - $info['dependencies'][] = $dependency; + + // If the project is Backdrop it won't have an info file to retrieve. + if ($project == 'backdrop') { + $info['type'] = 'core'; + } + else { + // If not Backdrop then retrieve the info file and process it. + // Compile the endpoint. + $endpoint = "$endpoint_base/contents/$project.info?ref=$ref"; + bee_instant_message('$endpoint:' . $endpoint, 'debug'); + // // Retrieve the info file as an array of lines. + $info_file = download_bee_github_api_call($endpoint, $github_api_token, 'raw'); + $info_parsed = bee_parse_info_format($info_file[0]); + bee_instant_message(bt('The info file data:'), 'debug', $info_parsed); + $info['type'] = $info_parsed['type']; + if (isset($info_parsed['dependencies'])) { + foreach ($info_parsed['dependencies'] as $dependency) { + $dependency = bee_parse_dependency($dependency); + $info['dependencies'][] = $dependency; + } } } @@ -700,9 +782,6 @@ function download_bee_select_version($project, $organization, $github_api_token return $result; } - - - /** * Download a project. * @@ -776,6 +855,13 @@ function download_bee_download_project($project, $source_url, $destination, $bra bee_instant_message('$temp/$directory:' . "$temp/$directory", 'debug'); } if ($replace) { + // If the project is Backdrop we only want to backup and/or replace the + // 'core' directory. + if ($project == 'backdrop') { + $destination .= '/core'; + $directory .= '/core'; + } + if ($backup) { $backup_destination = "$_bee_backdrop_root/" . BEE_BACKUP_DIRECTORY; $backup_destination .= bee_format_date($_SERVER['REQUEST_TIME'], $format = 'YmdHis') . '/'; @@ -1090,7 +1176,6 @@ function download_bee_github_api_call($endpoint, $github_api_token = '', $reques // Process the output according to the request type. switch ($request_type) { case 'raw': - // $output = explode("\n", $response); $output = array($response); break; default: diff --git a/docs/Usage.md b/docs/Usage.md index a749e4ad..a89387fc 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -116,20 +116,33 @@ options in a given file. - `bee projects` - Show information about all available projects. #### `download` -*Description:* Download Backdrop contrib projects together with dependencies. -*Aliases:* `dl` , `pm-download` +*Description:* Download Backdrop contrib projects. +*Aliases:* `dl`, `pm-download` *Arguments:* -- `projects` - One or more contrib projects to download. +- `projects` - One or more contrib projects to download. You can specify a release tag or branch for each project using the format: + project:release_tag/keyword[:branch] + + Specify a release tag or one of the following keywords: + - 'dev' (download the dev version from the default branch.) + - 'branch' (download the dev version from an alternative branch. Specified with + - 'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.) + + If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise. + + By default, 'latest' is selected if no release is specified. Dependencies will all download the 'latest' so if you do want a different version, download these first. *Options:* -- `--github-token` - A Github Personal Access Token (Classic) that can be used to extend the GitHub API rate. -- `--hide-progress`, `-h` - Deprecated, will get removed in a future version. -- `--allow-multisite-copy`, `-f` - Override the check that would prevent the project being downloaded to a multisite site if the project exists in the shared project directory. +- `--allow-multisite-copy`, `-f`` - Override the check that would prevent the project being downloaded to a multisite site if the project exists in the shared project directory. +- `--github-token=THE TOKEN.` - A Github Personal Access Token (Classic) that can be used to extend the GitHub API rate. *Examples:* -- `bee download webform` - Download the Webform module. +- `bee download webform` - Download the latest version of the Webform module. - `bee download simplify thesis bamboo` - Download the Simplify module, Thesis theme, and Bamboo layout. - `bee --site=site_name download simplify --allow-multisite-copy` - Download an additional copy of the Simplify module into the site_name multisite module folder. +- `bee download simplify:dev` - Download the dev version of the Simplify module. +- `bee download paragraphs:branch:1.x-1.2` - Download the 1.x-1.2 branch of the Paragraphs module +- `bee download simplify:1.x-1.2.4` - Download the 1.x-1.2.4 release of the Simplify module +- `bee download simplify:select` - Select from a list of releases and the dev version for the Simplify module. #### `enable` *Description:* Enable one or more projects (modules, themes, layouts). @@ -259,13 +272,20 @@ options in a given file. *Aliases:* `dl-core` *Arguments:* - `directory` - (optional) The directory to download and extract Backdrop into. Leave blank to use the current directory. + *Options:* -- `--github-token` - A Github Personal Access Token (Classic) that can be used to extend the GitHub API rate. -- `--hide-progress`, `-h` - Deprecated, will get removed in a future version. +- `--version=THE RELEASE TAG OR KEYWORD.` - Specify a release tag or one of the following keywords: + - 'dev' (download the dev version from the default branch.) + - 'branch' (download the dev version from an alternative branch. Specified by appending ':branch_name') + - 'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.) +- `--github-token=THE TOKEN.` - A Github Personal Access Token (Classic) that can be used to extend the GitHub API rate. *Examples:* - `bee download-core ../backdrop` - Download Backdrop into a 'backdrop' directory in the parent folder. - +- `bee download-core --version=1.28.3` - Download the 1.28.3 release of Backdrop into the current directory. +- `bee download-core --version=dev` - Download the dev version of Backdrop into the current directory. +- `bee download-core --version=branch:1.29.x` - Download the 1.29.x branch of Backdrop into the current directory. +- `bee download-core --version=select` - Select from a list of releases of Backdrop to download into the current directory. #### `install` *Description:* Install Backdrop and setup a new site. From 063b1e7f34626706bcda8712d256e0bb30d5783d Mon Sep 17 00:00:00 2001 From: Martin Price Date: Fri, 4 Oct 2024 11:41:53 +0100 Subject: [PATCH 07/15] Change bee_message to bee_instant_message where appropriate. --- commands/download.bee.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 05671591..7381dcbc 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -235,7 +235,7 @@ function download_bee_callback($arguments, $options) { if (in_array($dependency, $projects)) { // If project is already in the list to download, do not add to // the list to download again. - bee_message(bt("The '!dependency' !dependency_type is also required by the '!project' !project_type.", array( + bee_instant_message(bt("The '!dependency' !dependency_type is also required by the '!project' !project_type.", array( '!dependency' => !empty($submodule) ? "$dependency:$submodule" : $dependency, '!dependency_type' => (!empty($submodule) ? 'sub' : '') . $dependency_info['type'], '!project' => $project['name'], @@ -246,7 +246,7 @@ function download_bee_callback($arguments, $options) { // If project is not already in download list, then add to list // and prepare message. $projects[] = $dependency; - bee_message(bt("The '!dependency' !dependency_type will also be downloaded, as it is required by the '!project' !project_type.", array( + bee_instant_message(bt("The '!dependency' !dependency_type will also be downloaded, as it is required by the '!project' !project_type.", array( '!dependency' => !empty($submodule) ? "$dependency:$submodule" : $dependency, '!dependency_type' => (!empty($submodule) ? 'sub' : '') . $dependency_info['type'], '!project' => $project['name'], @@ -258,7 +258,7 @@ function download_bee_callback($arguments, $options) { // If project does exist in the site file system, give a // meaningful message so user is aware of the dependency and that // it is already met. - bee_message(bt("The '!dependency' !dependency_type is required by the '!project' !project_type but already exists at '!dependency_location'.", array( + bee_instant_message(bt("The '!dependency' !dependency_type is required by the '!project' !project_type but already exists at '!dependency_location'.", array( '!dependency' => $dependency, '!dependency_type' => $dependency_info['type'], '!project' => $project['name'], From 54cfa45dfad6fc21e674a5d9ab5ea5904cc55f84 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Sat, 19 Oct 2024 17:47:46 +0100 Subject: [PATCH 08/15] Progress to date --- commands/download.bee.inc | 77 ++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 7381dcbc..5f413047 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -372,6 +372,11 @@ function download_core_bee_callback($arguments, $options) { $info = download_bee_get_project_info('backdrop', 'backdrop', $core_version['release'], $core_version['branch'], $github_api_token); + if (!$info) { + bee_message(bt("Download operation is cancelled."), 'info'); + return; + } + // Get or create the directory to download Backdrop into. $destination = !empty($arguments['directory']) ? $arguments['directory'] : getcwd(); if (!realpath($destination)) { @@ -538,11 +543,17 @@ function download_bee_get_project_info($project, $organization, $release = '', $ $preferred_option = 'select'; $selection = download_bee_select_version($project, $organization, $github_api_token); - + if (empty($selection)) { + return FALSE; + } // Use the selection to set variables that will be used in the next step. if ($selection['value'] == 'dev') { $final_option = 'dev'; } + elseif ($selection['value'] == 'cancel') { + bee_message(bt("Download operation is cancelled."), 'info'); + return FALSE; + } else { $final_option = 'defined_release'; $defined_release = $selection['value']; @@ -563,23 +574,39 @@ function download_bee_get_project_info($project, $organization, $release = '', $ $release_data = download_bee_github_api_call($endpoint, $github_api_token); } else { - // If the release page does not exist offer to get the dev version - // instead. - $choice = bee_confirm(bt("The release you have specified (!release_tag) for '!organization/!project' does not exist. Do you want to select a version instead?", array( - '!release_tag' => $defined_release, - '!organization' => $organization, - '!project' => $project, - )), TRUE); + // If the release page does not exist offer to give them a choice of + // valid releases and the dev version instead. + if (is_numeric($defined_release[0])) { + $confirm_message = bt("The release you have specified (!release_tag) for '!organization/!project' does not exist. Do you want to select a version instead?", array( + '!release_tag' => $defined_release, + '!organization' => $organization, + '!project' => $project, + )); + } + else { + $confirm_message = bt("The version you specified (!release_tag) does not match a keyword or a release for '!organization/!project'. Do you want to select a version instead?", array( + '!release_tag' => $defined_release, + '!organization' => $organization, + '!project' => $project, + )); + } + $choice = bee_confirm($confirm_message, TRUE); if ($choice) { // If 'Yes' mode was enabled or user responded 'Yes', then continue // with selection. $selection = download_bee_select_version($project, $organization, $github_api_token); - + if (empty($selection)) { + return FALSE; + } // Use the selection to set variables that will be used in the next // step. if ($selection['value'] == 'dev') { $final_option = 'dev'; } + elseif ($selection['value'] == 'cancel') { + bee_message(bt("Download operation is cancelled."), 'info'); + return FALSE; + } else { $final_option = 'defined_release'; $defined_release = $selection['value']; @@ -599,7 +626,6 @@ function download_bee_get_project_info($project, $organization, $release = '', $ // Process the final option. switch ($final_option) { case 'latest': - // $download_url = "https://github.com/$organization/$project/releases/latest/download/$project.zip"; // Check GitHub API quota to ensure we can continue. if (!download_bee_check_github_api_quota($github_api_token)) { return FALSE; @@ -607,6 +633,14 @@ function download_bee_get_project_info($project, $organization, $release = '', $ // Get the latest release information. $endpoint = "$endpoint_base/releases/latest"; $release_data = download_bee_github_api_call($endpoint, $github_api_token); + if (empty($release_data)) { + // If release data can't be retrieved, display an error and return + // FALSE. + bee_message(bt("Release data for !project' could not be retrieved. Please try again later.", array( + '!project' => $project, + )), 'error'); + return FALSE; + } $defined_release = $release_data['tag_name']; bee_instant_message('$defined_release:' . $defined_release, 'debug'); // $defined_release_published_date = $latest_release['published_at']; @@ -693,6 +727,17 @@ function download_bee_get_project_info($project, $organization, $release = '', $ bee_instant_message('$endpoint:' . $endpoint, 'debug'); // // Retrieve the info file as an array of lines. $info_file = download_bee_github_api_call($endpoint, $github_api_token, 'raw'); + bee_instant_message('$info_file:', 'debug', $info_file); + // If the .info file cannot be found, then it will return a string + // containing a JSON object whereas if it can be found, it will return a + // string and therefore `json_decode` will return 'null'. + if (!is_null(json_decode($info_file[0]))) { + // If info file can't be retrieved, display an error and return FALSE. + bee_message(bt("Error retrieving .info file for '!project'.", array( + '!project' => $project, + )), 'error'); + return FALSE; + } $info_parsed = bee_parse_info_format($info_file[0]); bee_instant_message(bt('The info file data:'), 'debug', $info_parsed); $info['type'] = $info_parsed['type']; @@ -721,7 +766,7 @@ function download_bee_select_version($project, $organization, $github_api_token return FALSE; } // Get the list of releases. - $endpoint = "repos/$organization/$project/releases"; + $endpoint = "repos/$organization/$project/releases?per_page=10"; $releases = download_bee_github_api_call($endpoint, $github_api_token); // If one or more releases exist, then loop through them to get more // information. @@ -765,12 +810,15 @@ function download_bee_select_version($project, $organization, $github_api_token $message = bt("No releases exist but you can download the dev version."); $default = 'dev'; } + $select_options['cancel'] = 'Cancel'; // Present the choice to the user. $selection = bee_choice($select_options, $message, $default); + bee_instant_message('$selection: ' . $selection, 'debug'); switch ($selection) { case 'dev': + case 'cancel': $result = array( - 'value' => 'dev', + 'value' => $selection, ); break; default: @@ -779,6 +827,7 @@ function download_bee_select_version($project, $organization, $github_api_token 'data' => $releases_by_tag[$selection], ); } + bee_instant_message('$result: ', 'debug', $result); return $result; } @@ -1192,8 +1241,8 @@ function download_bee_github_api_call($endpoint, $github_api_token = '', $reques /** * Check whether a URL exists. * - * This helper function provides a simple check for URLs that does require any - * use of the GitHub API call quota. + * This helper function provides a simple check for URLs that does not require + * any use of the GitHub API call quota. * * @param string $url * The URL to check. From 9b2b2e7d09bf4ff1f639020c771306f20b5bb1e8 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Thu, 17 Apr 2025 18:39:08 +0100 Subject: [PATCH 09/15] Allow cancel from select menu. Handle negative choices. Adjust tests. --- commands/download.bee.inc | 26 +++++++++++++------ tests/backdrop/DownloadCommandsTest.php | 15 +++++++---- .../MultisiteDownloadCommandsTest.php | 12 ++++++--- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 5f413047..26ccc81e 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -182,6 +182,9 @@ function download_bee_callback($arguments, $options) { } else { $info = download_bee_get_project_info($project['name'], $organization, $project['release'], $project['branch'], $github_api_token); + if ($info === FALSE) { + return; + } if (empty($info)) { // If getting the info file has failed, show an error message here and // exit. @@ -216,6 +219,9 @@ function download_bee_callback($arguments, $options) { if (!$dependency_is_core) { // Get information about the dependency project. $dependency_info = download_bee_get_project_info($dependency, $organization, '', '', $github_api_token); + if ($dependency_info === FALSE) { + continue; + } if (empty($dependency_info)) { // If getting the info file has failed, show an error message here // and exit. @@ -304,9 +310,11 @@ function download_bee_callback($arguments, $options) { if (isset($info['release'])) { // If download is a release, whether latest or defined, show the // release tag and link to release notes. - bee_message(bt("'!project' (!release) was downloaded into '!directory'. For more information see !url", array( + bee_instant_message('release data: ', 'debug', $info['release']); + bee_message(bt("'!project' (!release, published at !release_date) was downloaded into '!directory'. For more information see !url", array( '!project' => $project['name'], '!release' => $info['release']['tag_name'], + '!release_date' => $info['release']['published_at'], '!url' => $info['release']['release_url'], '!directory' => $destination, )), 'success'); @@ -427,7 +435,8 @@ function download_core_bee_callback($arguments, $options) { * @param string $organization * The name of the GitHub organization. * @param string $release - * A string relating to a release option. Defaults to empty string (''): + * (Optional) A string relating to a release option. Defaults to empty string + * (''): * - 'dev': Download the dev version from the default branch. * - 'branch': Download the dev version from an alternative branch. Specified * using the $branch parameter. @@ -436,16 +445,17 @@ function download_core_bee_callback($arguments, $options) { * blank. If there is not a valid latest release, the dev branch will be * offered (see 'dev'). * - '1.x-1.2.3': The tag of a given release. If the release cannot be found - * an error message will be given and the function will reu. + * an error message will be given and the function will return FALSE. * - 'select': A list of valid options will be offered including: * - dev: (see dev). * - All releases that are not draft. Latest will be labelled and * pre-releases will also be labelled. * @param string $branch - * A string containing the name of the branch. Defaults to empty string (''). + * (Optional) A string containing the name of the branch. Defaults to empty + * string (''). * @param string $github_api_token - * The GitHub API Personal Access Token to apply to API calls. Defaults to - * empty string (''). + * (Optional) The GitHub API Personal Access Token to apply to API calls. + * Defaults to empty string (''). * * @return array|false * An associative array of information about the project, including all @@ -544,6 +554,7 @@ function download_bee_get_project_info($project, $organization, $release = '', $ $selection = download_bee_select_version($project, $organization, $github_api_token); if (empty($selection)) { + bee_message(bt("Download operation is cancelled."), 'info'); return FALSE; } // Use the selection to set variables that will be used in the next step. @@ -596,6 +607,7 @@ function download_bee_get_project_info($project, $organization, $release = '', $ // with selection. $selection = download_bee_select_version($project, $organization, $github_api_token); if (empty($selection)) { + bee_message(bt("Download operation is cancelled."), 'info'); return FALSE; } // Use the selection to set variables that will be used in the next @@ -643,8 +655,6 @@ function download_bee_get_project_info($project, $organization, $release = '', $ } $defined_release = $release_data['tag_name']; bee_instant_message('$defined_release:' . $defined_release, 'debug'); - // $defined_release_published_date = $latest_release['published_at']; - // $defined_release_url = $latest_release['html_url']; $download_url = "https://github.com/$organization/$project/releases/download/$defined_release/$project.zip"; bee_instant_message('$download_url:' . $download_url, 'debug'); $release_detail = array( diff --git a/tests/backdrop/DownloadCommandsTest.php b/tests/backdrop/DownloadCommandsTest.php index 21ac24a5..ebb0aee9 100644 --- a/tests/backdrop/DownloadCommandsTest.php +++ b/tests/backdrop/DownloadCommandsTest.php @@ -17,14 +17,17 @@ public function test_download_command_works() { global $bee_test_root; // Single module. $output_single = shell_exec('bee download simplify'); - $this->assertStringContainsString("'simplify' was downloaded into '$bee_test_root/backdrop/modules/simplify'.", (string) $output_single); + $pattern = '/\'simplify\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/backdrop\/modules\/simplify\'/'; + $this->assertRegExp($pattern, $output_single); $this->assertTrue(file_exists("$bee_test_root/backdrop/modules/simplify/simplify.info")); // Multiple projects (theme and layout). $output_multiple = shell_exec('bee download lumi bamboo'); - $this->assertStringContainsString("'lumi' was downloaded into '$bee_test_root/backdrop/themes/lumi'.", (string) $output_multiple); + $pattern = '/\'lumi\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/backdrop\/themes\/lumi\'/'; + $this->assertRegExp($pattern, $output_multiple); $this->assertTrue(file_exists("$bee_test_root/backdrop/themes/lumi/lumi.info")); - $this->assertStringContainsString("'bamboo' was downloaded into '$bee_test_root/backdrop/layouts/bamboo'.", (string) $output_multiple); + $pattern = '/\'bamboo\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/backdrop\/layouts\/bamboo\'/'; + $this->assertRegExp($pattern, $output_multiple); $this->assertTrue(file_exists("$bee_test_root/backdrop/layouts/bamboo/bamboo.info")); // Cleanup downloads. @@ -38,12 +41,14 @@ public function test_download_core_command_works() { global $bee_test_root; // Download to current directory. $output_current = shell_exec("mkdir $bee_test_root/current && cd $bee_test_root/current && bee download-core"); - $this->assertStringContainsString("Backdrop was downloaded into '$bee_test_root/current'.", (string) $output_current); + $pattern = '/Backdrop \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/current\'/'; + $this->assertRegExp($pattern, $output_current); $this->assertTrue(file_exists("$bee_test_root/current/index.php")); // Download to specified directory. $output_directory = shell_exec("bee download-core $bee_test_root/directory"); - $this->assertStringContainsString("Backdrop was downloaded into '$bee_test_root/directory'.", (string) $output_directory); + $pattern = '/Backdrop \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/directory\'/'; + $this->assertRegExp($pattern, $output_directory); $this->assertTrue(file_exists("$bee_test_root/directory/index.php")); // Cleanup downloads. diff --git a/tests/multisite/MultisiteDownloadCommandsTest.php b/tests/multisite/MultisiteDownloadCommandsTest.php index c45dbccf..68af2e51 100644 --- a/tests/multisite/MultisiteDownloadCommandsTest.php +++ b/tests/multisite/MultisiteDownloadCommandsTest.php @@ -17,7 +17,8 @@ public function test_download_command_works() { global $bee_test_root; // Root directory, no site specified. $output_root = shell_exec('bee download simplify'); - $this->assertStringContainsString("'simplify' was downloaded into '$bee_test_root/multisite/modules/simplify'.", (string) $output_root); + $pattern = '/\'simplify\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/multisite\/modules\/simplify\'/'; + $this->assertRegExp($pattern, $output_root); $this->assertTrue(file_exists("$bee_test_root/multisite/modules/simplify/simplify.info")); // Root directory, site specified, 'allow-multisite-copy' option NOT @@ -27,17 +28,20 @@ public function test_download_command_works() { // Root directory, site specified, 'allow-multisite-copy' option included. $output_root = shell_exec('bee --site=multi_one download --allow-multisite-copy simplify'); - $this->assertStringContainsString("'simplify' was downloaded into '$bee_test_root/multisite/sites/multi_one/modules/simplify'.", (string) $output_root); + $pattern = '/\'simplify\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/multisite\/sites\/multi_one\/modules\/simplify\'/'; + $this->assertRegExp($pattern, $output_root); $this->assertTrue(file_exists("$bee_test_root/multisite/sites/multi_one/modules/simplify/simplify.info")); // Root directory, site specified. $output_root_site = shell_exec('bee download --site=multi_one lumi'); - $this->assertStringContainsString("'lumi' was downloaded into '$bee_test_root/multisite/sites/multi_one/themes/lumi'.", (string) $output_root_site); + $pattern = '/\'lumi\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/multisite\/sites\/multi_one\/themes\/lumi\'/'; + $this->assertRegExp($pattern, $output_root_site); $this->assertTrue(file_exists("$bee_test_root/multisite/sites/multi_one/themes/lumi/lumi.info")); // Site directory. $output_site = shell_exec('cd sites/multi_two && bee download bamboo'); - $this->assertStringContainsString("'bamboo' was downloaded into '$bee_test_root/multisite/sites/multi_two/layouts/bamboo'.", (string) $output_site); + $pattern = '/\'bamboo\' \([\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/multisite\/sites\/multi_two\/layouts\/bamboo\'/'; + $this->assertRegExp($pattern, $output_site); $this->assertTrue(file_exists("$bee_test_root/multisite/sites/multi_two/layouts/bamboo/bamboo.info")); // Cleanup downloads. From 9340748bc8f8f1a3f01f57580e7f992b5b9217db Mon Sep 17 00:00:00 2001 From: Martin Price Date: Thu, 17 Apr 2025 22:38:38 +0100 Subject: [PATCH 10/15] Update the Changelog with changes added elsewhere. --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f5156e..4c619b23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project follows the which is based on the major version of Backdrop CMS with a semantic version system for each contributed module, theme and layout. -## [Unreleased] - 2024-TBC +## [Unreleased] - 2025-04-17 ### Added - An option for the `db-import` command to allow import from newer MariaDB @@ -18,13 +18,16 @@ database or client does not support it. - The ability to download specified releases or branches of modules, themes, layouts or Backdrop itself. +### Fixed +- Unhandled errors and warnings if commands run outside Backdrop root and/or +before installing Backdrop. + ### Changed +- Bee will now notify the user of additional modules that will be enabled or disabled based on module dependencies when using the enable and disable commands. - The functions within the download command have been made more flexible and better able to support the coming 'update' command. -### Fixed -- Unhandled errors and warnings if commands run outside Backdrop root and/or -before installing Backdrop. + ## [1.x-1.1.0] - 2024-09-07 From 360f0527aac27adb89de562920c3318e04905ab7 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Sat, 19 Apr 2025 17:22:12 +0100 Subject: [PATCH 11/15] Move check for exsting project before determine which version to download --- commands/download.bee.inc | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index 26ccc81e..dffa27ca 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -172,6 +172,24 @@ function download_bee_callback($arguments, $options) { $project['branch'] = ($project['release'] == 'branch') ? $project_details[2] : ''; } + // Check if the project exists within the site file system. It is desirable + // to do this check before doing checks about what version to download. At + // this stage, we don't know if it is a module, theme or layout template so + // we need to check each but can exit if found. + $types = array('module', 'theme', 'layout'); + foreach ($types as $type) { + $project_existing_location = download_bee_check_project_exists($project['name'], $type); + $project_allow_multisite_copy = download_bee_check_multisite_copy($allow_multisite_copy, $project_existing_location); + if ($project_existing_location != FALSE && !$project_allow_multisite_copy) { + bee_message(bt("'!project' already exists in '!existing_location'.", array( + '!project' => $project['name'], + '!existing_location' => $project_existing_location, + )), 'error'); + $project_count++; + continue 2; + } + } + // Check if the project exists by trying to get the repo homepage. $organization = 'backdrop-contrib'; $url = "https://github.com/$organization/" . $project['name']; @@ -183,7 +201,8 @@ function download_bee_callback($arguments, $options) { else { $info = download_bee_get_project_info($project['name'], $organization, $project['release'], $project['branch'], $github_api_token); if ($info === FALSE) { - return; + $project_count++; + continue; } if (empty($info)) { // If getting the info file has failed, show an error message here and @@ -191,7 +210,8 @@ function download_bee_callback($arguments, $options) { bee_message(bt("The info file for !project cannot be retrieved at this time.", array( '!project' => $project['name'], )), 'error'); - return; + $project_count++; + continue; } // Get the list of dependencies and add to list of projects. if (!empty($info['dependencies'])) { @@ -281,22 +301,14 @@ function download_bee_callback($arguments, $options) { bee_message(bt("The 'type' of project '!project' could not be determined.", array( '!project' => $project['name'], )), 'error'); - return; + $project_count++; + continue; } + // Check that the destination can be created. $destination = download_bee_get_destination_path($project['name'], $info['type']); bee_instant_message('$destination:' . $destination, 'debug'); - // Check if the project exists within the site file system. - $project_existing_location = download_bee_check_project_exists($project['name'], $info['type']); - $project_allow_multisite_copy = download_bee_check_multisite_copy($allow_multisite_copy, $project_existing_location); - if ($project_existing_location != FALSE && !$project_allow_multisite_copy) { - bee_message(bt("'!project' already exists in '!existing_location'.", array( - '!project' => $project['name'], - '!existing_location' => $project_existing_location, - )), 'error'); - return; - } - elseif (!mkdir($destination, 0755, TRUE)) { + if (!mkdir($destination, 0755, TRUE)) { bee_message(bt("The destination directory '!directory' doesn't exist and couldn't be created.", array( '!directory' => $destination, )), 'error'); From 061ecb43a26d69c39d028d4ab835ac33927b2851 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Sat, 19 Apr 2025 17:27:09 +0100 Subject: [PATCH 12/15] Clarify changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf85d398..e14c0aab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ servers with the enable sandbox command in the dump file if the destination database or client does not support it. - Tooling for the lando recipe to support Xdebug with VS Code. - The ability to download specified releases or branches of modules, themes, -layouts or Backdrop itself. +layout templates or Backdrop itself. ### Fixed - Unhandled errors and warnings if commands run outside Backdrop root and/or From ec2857c59fdac452794ae27e6c02ea61bd06ea96 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Sat, 19 Apr 2025 21:07:43 +0100 Subject: [PATCH 13/15] Add tests for new options. Add release date to message for dl-core --- .lando/setup.sh | 2 +- commands/download.bee.inc | 5 +++-- tests/backdrop/DownloadCommandsTest.php | 13 +++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.lando/setup.sh b/.lando/setup.sh index f6656a75..b1c6b985 100755 --- a/.lando/setup.sh +++ b/.lando/setup.sh @@ -8,7 +8,7 @@ clean_up() { cd /app # Remove Backdrop installations. - rm -rf backdrop/ multisite/ current/ directory/ + rm -rf backdrop/ multisite/ current/ directory/ defined_release/ # Drop databases. mysql -h database -u root -e "DROP DATABASE backdrop;" diff --git a/commands/download.bee.inc b/commands/download.bee.inc index dffa27ca..df479933 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -391,7 +391,7 @@ function download_core_bee_callback($arguments, $options) { } $info = download_bee_get_project_info('backdrop', 'backdrop', $core_version['release'], $core_version['branch'], $github_api_token); - + bee_instant_message('project_info:', 'debug', $info); if (!$info) { bee_message(bt("Download operation is cancelled."), 'info'); return; @@ -422,9 +422,10 @@ function download_core_bee_callback($arguments, $options) { if (isset($info['release'])) { // If download is a release, whether latest or defined, show the // release tag and link to release notes. - bee_message(bt("Backdrop (!release) was downloaded into '!directory'. For more information see !url", array( + bee_message(bt("Backdrop (!release, published at !release_date) was downloaded into '!directory'. For more information see !url", array( '!release' => $info['release']['tag_name'], '!url' => $info['release']['release_url'], + '!release_date' => $info['release']['published_at'], '!directory' => $destination, )), 'success'); } diff --git a/tests/backdrop/DownloadCommandsTest.php b/tests/backdrop/DownloadCommandsTest.php index ebb0aee9..18e0384c 100644 --- a/tests/backdrop/DownloadCommandsTest.php +++ b/tests/backdrop/DownloadCommandsTest.php @@ -30,8 +30,11 @@ public function test_download_command_works() { $this->assertRegExp($pattern, $output_multiple); $this->assertTrue(file_exists("$bee_test_root/backdrop/layouts/bamboo/bamboo.info")); + // Defined release. + $output_defined_release = shell_exec('bee download layout_custom_theme:1.x-1.0.4'); + $pattern = '/\'layout_custom_theme\' \(1\.x\-1\.0\.4\, published at 2024\-02\-01T[\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/backdrop\/modules\/layout_custom_theme\'/'; // Cleanup downloads. - exec("rm -fr $bee_test_root/backdrop/modules/simplify $bee_test_root/backdrop/themes/lumi $bee_test_root/backdrop/layouts/bamboo"); + exec("rm -fr $bee_test_root/backdrop/modules/simplify $bee_test_root/backdrop/themes/lumi $bee_test_root/backdrop/layouts/bamboo $bee_test_root/backdrop/modules/layout_custom_theme"); } /** @@ -51,8 +54,14 @@ public function test_download_core_command_works() { $this->assertRegExp($pattern, $output_directory); $this->assertTrue(file_exists("$bee_test_root/directory/index.php")); + // Download a defined release. + $output_defined_release = shell_exec("bee download-core $bee_test_root/defined_release --version=1.30.0"); + $pattern = '/Backdrop \(1\.30\.0\, published at 2025\-01\-1\dT[\w\s\.\W]*\) was downloaded into \'' . preg_quote($bee_test_root, '/') . '\/defined_release\'/'; + $this->assertRegExp($pattern, $output_defined_release); + $this->assertTrue(file_exists("$bee_test_root/defined_release/index.php")); + // Cleanup downloads. - exec("rm -fr $bee_test_root/current $bee_test_root/directory"); + exec("rm -fr $bee_test_root/current $bee_test_root/directory $bee_test_root/defined_release"); } } From e108b143628330c485f2ca8907157c5c34892b84 Mon Sep 17 00:00:00 2001 From: Martin Price Date: Sat, 19 Apr 2025 22:30:30 +0100 Subject: [PATCH 14/15] Fix typo --- docs/Usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Usage.md b/docs/Usage.md index a89387fc..42e867f9 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -132,7 +132,7 @@ options in a given file. By default, 'latest' is selected if no release is specified. Dependencies will all download the 'latest' so if you do want a different version, download these first. *Options:* -- `--allow-multisite-copy`, `-f`` - Override the check that would prevent the project being downloaded to a multisite site if the project exists in the shared project directory. +- `--allow-multisite-copy`, `-f` - Override the check that would prevent the project being downloaded to a multisite site if the project exists in the shared project directory. - `--github-token=THE TOKEN.` - A Github Personal Access Token (Classic) that can be used to extend the GitHub API rate. *Examples:* From ff136fdc959eeaacf5fa675eeab9236296cf0a5f Mon Sep 17 00:00:00 2001 From: Martin Price Date: Mon, 21 Apr 2025 11:02:46 +0100 Subject: [PATCH 15/15] Fixed missing part of argument description for branch --- commands/download.bee.inc | 2 +- docs/Usage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/download.bee.inc b/commands/download.bee.inc index df479933..b974c7e5 100644 --- a/commands/download.bee.inc +++ b/commands/download.bee.inc @@ -17,7 +17,7 @@ function download_bee_command() { $projects_description .= "\n " . bt('project:release_tag/keyword[:branch]'); $projects_description .= "\n\n " . bt("Specify a release tag or one of the following keywords:"); $projects_description .= "\n - " . bt("'dev' (download the dev version from the default branch.)"); - $projects_description .= "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified with "); + $projects_description .= "\n - " . bt("'branch' (download the dev version from an alternative branch. Specified with ':branch_name'."); $projects_description .= "\n - " . bt("'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.)"); $projects_description .= "\n\n " . bt("If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise."); $projects_description .= "\n\n " . bt("By default, 'latest' is selected if no release is specified. Dependencies will all download the 'latest' so if you do want a different version, download these first."); diff --git a/docs/Usage.md b/docs/Usage.md index 42e867f9..1be0ef1a 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -124,7 +124,7 @@ options in a given file. Specify a release tag or one of the following keywords: - 'dev' (download the dev version from the default branch.) - - 'branch' (download the dev version from an alternative branch. Specified with + - 'branch' (download the dev version from an alternative branch. Specified with `:branch_name`.) - 'select' (a list of valid options will be offered including dev and all releases that are not draft. Latest and pre-releases will be labelled.) If 'branch' is entered for release, then the name of an alternative branch can be entered with this option. It is ignored otherwise.