From 66bde686f6b7d97eb937c0eeb729c451d4770dca Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 24 Mar 2025 12:31:26 -0500 Subject: [PATCH 01/34] Change grades url to ngrok for testing --- README.markdown | 2 +- Rakefile | 2 +- grade_runner.gemspec | 3 +-- lib/tasks/grade.rake | 12 ++++++------ lib/tasks/grade_runner.rake | 2 +- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.markdown b/README.markdown index f19897c..116beab 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,6 @@ # grade_runner -A Ruby client for [firstdraft Grades](https://grades.firstdraft.com) +A Ruby client for [Grades](https://f5cd-98-227-60-153.ngrok-free.app) ## Installation diff --git a/Rakefile b/Rakefile index 1d612ce..d942d71 100644 --- a/Rakefile +++ b/Rakefile @@ -16,7 +16,7 @@ Juwelier::Tasks.new do |gem| gem.name = "grade_runner" gem.homepage = "http://github.com/firstdraft/grade_runner" gem.license = "MIT" - gem.summary = %Q{A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)} + gem.summary = %Q{A Ruby client for [firstdraft Grades](https://f5cd-98-227-60-153.ngrok-free.app)} gem.description = %Q{This gem runs your RSpec test suite and posts the JSON output to grades.firstdraft.com.} gem.email = ["raghu@firstdraft.com", "jelani@firstdraft.com"] gem.authors = ["Raghu Betina", "Jelani Woods"] diff --git a/grade_runner.gemspec b/grade_runner.gemspec index 0c171a9..340bda6 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -38,7 +38,7 @@ Gem::Specification.new do |s| s.homepage = "http://github.com/firstdraft/grade_runner".freeze s.licenses = ["MIT".freeze] s.rubygems_version = "3.1.6".freeze - s.summary = "A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)".freeze + s.summary = "A Ruby client for [firstdraft Grades](https://f5cd-98-227-60-153.ngrok-free.app)".freeze if s.respond_to? :specification_version then s.specification_version = 4 @@ -82,4 +82,3 @@ Gem::Specification.new do |s| s.add_dependency(%q.freeze, ["~> 0"]) end end - diff --git a/lib/tasks/grade.rake b/lib/tasks/grade.rake index 34114a8..87b6fd9 100644 --- a/lib/tasks/grade.rake +++ b/lib/tasks/grade.rake @@ -20,7 +20,7 @@ namespace :grade do config_dir_name = find_or_create_directory(".vscode") config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" student_config = {} - student_config["submission_url"] = "https://grades.firstdraft.com" + student_config["submission_url"] = "https://f5cd-98-227-60-153.ngrok-free.app" student_config["github_username"] = retrieve_github_username if File.exist?(config_file_name) @@ -33,7 +33,7 @@ namespace :grade do file_token = config["personal_access_token"] student_config["submission_url"] = config["submission_url"] else - submission_url = "https://grades.firstdraft.com" + submission_url = "https://f5cd-98-227-60-153.ngrok-free.app" end if file_token.nil? && ENV.has_key?("LTICI_GITPOD_APITOKEN") input_token = ENV.fetch("LTICI_GITPOD_APITOKEN") @@ -63,14 +63,14 @@ namespace :grade do end end end - - if token.present? + + if token.present? if is_valid_token?(submission_url, token) == false student_config["personal_access_token"] = nil update_config_file(config_file_name, student_config) puts "Your access token looked invalid, so we've reset it to be blank. Please re-run rails grade and, when asked, copy-paste your token carefully from the assignment page." - else + else if GradeRunner.override_local_specs resource_info = upstream_repo(submission_url, token) full_reponame = resource_info.fetch("repo_slug") @@ -111,7 +111,7 @@ namespace :grade do task :reset_token do config_dir_name = find_or_create_directory(".vscode") config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - submission_url = "https://grades.firstdraft.com" + submission_url = "https://f5cd-98-227-60-153.ngrok-free.app" student_config = {} student_config["submission_url"] = submission_url diff --git a/lib/tasks/grade_runner.rake b/lib/tasks/grade_runner.rake index ddde8a2..6ade520 100644 --- a/lib/tasks/grade_runner.rake +++ b/lib/tasks/grade_runner.rake @@ -1,7 +1,7 @@ namespace :grade_runner do desc "Grade project" task :runner do - default_submission_url = "https://grades.firstdraft.com" + default_submission_url = "https://f5cd-98-227-60-153.ngrok-free.app" config = {} path = Rails.root.join("grades.yml") if File.exist?(path) From c08fbc3875e611a0b33973db464c654345d5635f Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 24 Mar 2025 16:07:00 -0500 Subject: [PATCH 02/34] Disable token regex validation for now --- lib/tasks/grade.rake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tasks/grade.rake b/lib/tasks/grade.rake index 87b6fd9..59ba300 100644 --- a/lib/tasks/grade.rake +++ b/lib/tasks/grade.rake @@ -218,7 +218,9 @@ def find_or_create_directory(directory_name) end def is_valid_token?(root_url, token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ + return false unless token.is_a?(String) + # TODO: add custom token creation + # && token =~ /^[1-9A-Za-z][^OIl]{23}$/ url = "#{root_url}/submissions/validate_token?token=#{token}" uri = URI.parse(url) req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') From 7e4f2287f10a356571585698798b65543ecb2962 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Tue, 25 Mar 2025 14:42:29 -0500 Subject: [PATCH 03/34] Comment out spec validation code for now. --- lib/tasks/grade.rake | 133 +++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 67 deletions(-) diff --git a/lib/tasks/grade.rake b/lib/tasks/grade.rake index 59ba300..f236fdd 100644 --- a/lib/tasks/grade.rake +++ b/lib/tasks/grade.rake @@ -77,7 +77,8 @@ namespace :grade do remote_spec_folder_sha = resource_info.fetch("spec_folder_sha") source_code_url = resource_info.fetch("source_code_url") set_upstream_remote(full_reponame) - sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) + # TODO: verify specs match source code + # sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) end path = File.join(project_root, "/tmp/output/#{Time.now.to_i}.json") @@ -95,17 +96,17 @@ namespace :grade do end end - desc "Run only the next failing test." - task :next do - path = File.join(project_root, "examples.txt") - if File.exist?(path) - `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - puts `RAILS_ENV=test bundle exec rspec --next-failure --format HintFormatter` - else - puts `RAILS_ENV=test bundle exec rspec` - puts "Please rerun rails grade:next to run the first failing spec" - end - end + # desc "Run only the next failing test." + # task :next do + # path = File.join(project_root, "examples.txt") + # if File.exist?(path) + # `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) + # puts `RAILS_ENV=test bundle exec rspec --next-failure --format HintFormatter` + # else + # puts `RAILS_ENV=test bundle exec rspec` + # puts "Please rerun rails grade:next to run the first failing spec" + # end + # end desc "Reset access token saved in YAML file." task :reset_token do @@ -139,64 +140,64 @@ namespace :grade do end -def sync_specs_with_source(full_reponame, remote_sha, repo_url) - # Unstage staged changes in spec folder - `git restore --staged spec/* ` - # Discard unstaged changes in spec folder - `git checkout spec -q` - `git clean spec -f -q` - local_sha = `git ls-tree HEAD #{project_root.join('spec')}`.chomp.split[2] +# def sync_specs_with_source(full_reponame, remote_sha, repo_url) +# # Unstage staged changes in spec folder +# `git restore --staged spec/* ` +# # Discard unstaged changes in spec folder +# `git checkout spec -q` +# `git clean spec -f -q` +# local_sha = `git ls-tree HEAD #{project_root.join('spec')}`.chomp.split[2] - unless remote_sha == local_sha - find_or_create_directory("tmp") - find_or_create_directory("tmp/backup") - files_and_subfolders_inside_specs = Dir.glob("spec/*") - # Temporarily move specs - FileUtils.mv(files_and_subfolders_inside_specs, "tmp/backup") +# unless remote_sha == local_sha +# find_or_create_directory("tmp") +# find_or_create_directory("tmp/backup") +# files_and_subfolders_inside_specs = Dir.glob("spec/*") +# # Temporarily move specs +# FileUtils.mv(files_and_subfolders_inside_specs, "tmp/backup") - download_file(repo_url, "tmp/spec.zip") - extracted_zip_folder = extract_zip("tmp/spec.zip", "tmp") - source_directory = extracted_zip_folder.join("spec") - overwrite_spec_folder(source_directory) +# download_file(repo_url, "tmp/spec.zip") +# extracted_zip_folder = extract_zip("tmp/spec.zip", "tmp") +# source_directory = extracted_zip_folder.join("spec") +# overwrite_spec_folder(source_directory) - FileUtils.rm(project_root.join("tmp/spec.zip")) - FileUtils.rm_rf(extracted_zip_folder) - FileUtils.rm_rf("tmp/backup") - `git add spec/` - `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft "` - end -end +# FileUtils.rm(project_root.join("tmp/spec.zip")) +# FileUtils.rm_rf(extracted_zip_folder) +# FileUtils.rm_rf("tmp/backup") +# `git add spec/` +# `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft "` +# end +# end -def download_file(url, destination) - download = URI.open(url) - IO.copy_stream(download, destination) -end +# def download_file(url, destination) +# download = URI.open(url) +# IO.copy_stream(download, destination) +# end -def extract_zip(folder, destination) - extracted_file_path = project_root.join(destination) - Zip::File.open(folder) do |zip_file| - zip_file.each_with_index do |file, index| - # Get name of root folder in zip file - if index == 0 - extracted_file_path = extracted_file_path.join(file.name) - end - file_path = File.join(destination, file.name) - FileUtils.mkdir_p(File.dirname(file_path)) - file.extract(file_path) - end - end - extracted_file_path -end +# def extract_zip(folder, destination) +# extracted_file_path = project_root.join(destination) +# Zip::File.open(folder) do |zip_file| +# zip_file.each_with_index do |file, index| +# # Get name of root folder in zip file +# if index == 0 +# extracted_file_path = extracted_file_path.join(file.name) +# end +# file_path = File.join(destination, file.name) +# FileUtils.mkdir_p(File.dirname(file_path)) +# file.extract(file_path) +# end +# end +# extracted_file_path +# end -def overwrite_spec_folder(source_directory) - destination_directory = "spec" - # Get all files in the source directory - files = Dir.glob("#{source_directory}/*") - # Move each file to the destination directory - files.each do |file| - FileUtils.mv(file, destination_directory) - end -end +# def overwrite_spec_folder(source_directory) +# destination_directory = "spec" +# # Get all files in the source directory +# files = Dir.glob("#{source_directory}/*") +# # Move each file to the destination directory +# files.each do |file| +# FileUtils.mv(file, destination_directory) +# end +# end def set_upstream_remote(repo_slug) upstream = `git remote -v | grep -w upstream`.chomp @@ -218,9 +219,7 @@ def find_or_create_directory(directory_name) end def is_valid_token?(root_url, token) - return false unless token.is_a?(String) - # TODO: add custom token creation - # && token =~ /^[1-9A-Za-z][^OIl]{23}$/ + return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ url = "#{root_url}/submissions/validate_token?token=#{token}" uri = URI.parse(url) req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') From cdb11b777db94d143b82b6358d3312e5197284b2 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Wed, 2 Apr 2025 17:29:47 -0500 Subject: [PATCH 04/34] Refactor grade runner into services and utils and make submission url configurable. --- README.markdown | 9 +- Rakefile | 2 +- grade_runner.gemspec | 2 +- lib/grade_runner.rb | 18 +- lib/grade_runner/runner.rb | 4 +- lib/grade_runner/services/config_service.rb | 55 ++++ lib/grade_runner/services/github_service.rb | 49 ++++ lib/grade_runner/services/grade_service.rb | 79 ++++++ lib/grade_runner/services/spec_service.rb | 33 +++ lib/grade_runner/services/token_service.rb | 70 ++++++ lib/grade_runner/utils/path_utils.rb | 42 ++++ lib/tasks/grade.rake | 266 +------------------- lib/tasks/grade_runner.rake | 21 +- 13 files changed, 374 insertions(+), 276 deletions(-) create mode 100644 lib/grade_runner/services/config_service.rb create mode 100644 lib/grade_runner/services/github_service.rb create mode 100644 lib/grade_runner/services/grade_service.rb create mode 100644 lib/grade_runner/services/spec_service.rb create mode 100644 lib/grade_runner/services/token_service.rb create mode 100644 lib/grade_runner/utils/path_utils.rb diff --git a/README.markdown b/README.markdown index 116beab..30ca05a 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,6 @@ # grade_runner -A Ruby client for [Grades](https://f5cd-98-227-60-153.ngrok-free.app) +A Ruby client for [Grades](https://grades.firstdraft.com) ## Installation @@ -48,6 +48,7 @@ if Rails.env.development? || Rails.env.test? GradeRunner.config do |config| config.default_points = 1 # default 1 config.override_local_specs = false # default true + config.submission_url = "https://your-grading-server-url.com" # Set custom submission URL end end ``` @@ -91,6 +92,12 @@ require "rubygems" require "bundler/setup" require "rake" +# Configure grade_runner (optional) +require "grade_runner" +GradeRunner.config do |config| + config.submission_url = "https://your-grading-server-url.com" # Set custom submission URL +end + dir = Gem::Specification.find_by_name("grade_runner").gem_dir load "#{dir}/lib/tasks/grade.rake" diff --git a/Rakefile b/Rakefile index d942d71..1d612ce 100644 --- a/Rakefile +++ b/Rakefile @@ -16,7 +16,7 @@ Juwelier::Tasks.new do |gem| gem.name = "grade_runner" gem.homepage = "http://github.com/firstdraft/grade_runner" gem.license = "MIT" - gem.summary = %Q{A Ruby client for [firstdraft Grades](https://f5cd-98-227-60-153.ngrok-free.app)} + gem.summary = %Q{A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)} gem.description = %Q{This gem runs your RSpec test suite and posts the JSON output to grades.firstdraft.com.} gem.email = ["raghu@firstdraft.com", "jelani@firstdraft.com"] gem.authors = ["Raghu Betina", "Jelani Woods"] diff --git a/grade_runner.gemspec b/grade_runner.gemspec index 340bda6..29e1675 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -38,7 +38,7 @@ Gem::Specification.new do |s| s.homepage = "http://github.com/firstdraft/grade_runner".freeze s.licenses = ["MIT".freeze] s.rubygems_version = "3.1.6".freeze - s.summary = "A Ruby client for [firstdraft Grades](https://f5cd-98-227-60-153.ngrok-free.app)".freeze + s.summary = "A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)".freeze if s.respond_to? :specification_version then s.specification_version = 4 diff --git a/lib/grade_runner.rb b/lib/grade_runner.rb index c9816bd..b77eab8 100644 --- a/lib/grade_runner.rb +++ b/lib/grade_runner.rb @@ -1,11 +1,19 @@ require "grade_runner/runner" +require "grade_runner/utils/path_utils" +require "grade_runner/services/token_service" +require "grade_runner/services/config_service" +require "grade_runner/services/github_service" +require "grade_runner/services/spec_service" +require "grade_runner/services/grade_service" require "grade_runner/railtie" if defined?(Rails) module GradeRunner class Error < StandardError; end + DEFAULT_SUBMISSION_URL = "https://grades.firstdraft.com" + class << self - attr_writer :default_points, :override_local_specs + attr_writer :default_points, :override_local_specs, :submission_url def default_points @default_points || 1 @@ -19,8 +27,16 @@ def override_local_specs end end + def submission_url + @submission_url || DEFAULT_SUBMISSION_URL + end + def config yield self end + + def project_root + Utils::PathUtils.project_root + end end end diff --git a/lib/grade_runner/runner.rb b/lib/grade_runner/runner.rb index a21a243..bcc5d5c 100644 --- a/lib/grade_runner/runner.rb +++ b/lib/grade_runner/runner.rb @@ -3,7 +3,6 @@ module GradeRunner class Runner - def initialize(submission_root_url, grades_access_token, rspec_output_json, username, reponame, sha, source) @submission_url = submission_root_url + submission_path @grades_access_token = grades_access_token @@ -47,6 +46,5 @@ def data source: @source } end - end -end +end \ No newline at end of file diff --git a/lib/grade_runner/services/config_service.rb b/lib/grade_runner/services/config_service.rb new file mode 100644 index 0000000..bab3224 --- /dev/null +++ b/lib/grade_runner/services/config_service.rb @@ -0,0 +1,55 @@ +require "yaml" +require "fileutils" +require "grade_runner/utils/path_utils" + +module GradeRunner + module Services + class ConfigService + def initialize + @path_utils = GradeRunner::Utils::PathUtils + end + + def find_or_create_directory(directory_name) + @path_utils.find_or_create_directory(directory_name) + end + + def update_config_file(config_file_name, config) + File.write(config_file_name, YAML.dump(config)) + end + + def load_config(config_file_name, default_submission_url) + if File.exist?(config_file_name) + begin + YAML.load_file(config_file_name) + rescue + abort "It looks like there's something wrong with your token in `#{config_file_name}`. Please delete that file and try `rails grade` again, and be sure to provide the access token for THIS project.".red + end + else + { "submission_url" => default_submission_url } + end + end + + def get_config_file_path(directory = ".vscode", filename = ".ltici_apitoken.yml") + config_dir = find_or_create_directory(directory) + "#{config_dir}/#{filename}" + end + + def save_token_to_config(config_file_name, token, submission_url, github_username) + config = { + "submission_url" => submission_url, + "personal_access_token" => token, + "github_username" => github_username + } + update_config_file(config_file_name, config) + end + + def clear_token_in_config(config_file_name) + if File.exist?(config_file_name) + config = YAML.load_file(config_file_name) + config["personal_access_token"] = nil + update_config_file(config_file_name, config) + end + end + end + end +end \ No newline at end of file diff --git a/lib/grade_runner/services/github_service.rb b/lib/grade_runner/services/github_service.rb new file mode 100644 index 0000000..e2e66d9 --- /dev/null +++ b/lib/grade_runner/services/github_service.rb @@ -0,0 +1,49 @@ +require "octokit" +require "grade_runner/utils/path_utils" + +module GradeRunner + module Services + class GithubService + def initialize + @path_utils = GradeRunner::Utils::PathUtils + end + + def retrieve_github_username(config_file_name) + if File.exist?(config_file_name) + config = YAML.load_file(config_file_name) + return config["github_username"] if config["github_username"].present? + end + + github_email = `git config user.email`.chomp + return "" if github_email.blank? + + username = `git config user.name`.chomp + search_results = Octokit.search_users("#{github_email} in:email").fetch(:items) + + if search_results.present? + username = search_results.first.fetch(:login, username) + end + + username + end + + def set_upstream_remote(repo_slug) + upstream = `git remote -v | grep -w upstream`.chomp + if upstream.blank? + `git remote add upstream https://github.com/#{repo_slug}` + else + `git remote set-url upstream https://github.com/#{repo_slug}` + end + end + + def get_commit_sha + `git rev-parse HEAD`.slice(0..7) + end + + def get_repo_name + # Extract the repository name from the project path + @path_utils.project_root.to_s.split("/").last + end + end + end +end \ No newline at end of file diff --git a/lib/grade_runner/services/grade_service.rb b/lib/grade_runner/services/grade_service.rb new file mode 100644 index 0000000..4a21cd8 --- /dev/null +++ b/lib/grade_runner/services/grade_service.rb @@ -0,0 +1,79 @@ +module GradeRunner + module Services + class GradeService + def initialize + @submission_url = GradeRunner.submission_url + @config_service = ConfigService.new + @token_service = TokenService.new(@submission_url) + @github_service = GithubService.new + @spec_service = SpecService.new + end + + def process_grade_all(input_token = nil) + config_file_name = @config_service.get_config_file_path + config = @config_service.load_config(config_file_name, @submission_url) + + file_token = config["personal_access_token"] + + # Check for Gitpod environment token + if file_token.nil? && ENV.has_key?("LTICI_GITPOD_APITOKEN") + input_token = ENV.fetch("LTICI_GITPOD_APITOKEN") + end + + # Get token (from input, file, or prompt) + token = @token_service.get_token(input_token, file_token, config_file_name) + github_username = @github_service.retrieve_github_username(config_file_name) + + # Save token to config if new + if input_token.present? || (token.present? && file_token.nil?) + @config_service.save_token_to_config(config_file_name, token, @submission_url, github_username) + end + + return "We couldn't find your access token. Please click on the assignment link and run the rails grade command shown there." if token.blank? + + # Validate token + if !@token_service.validate_token(token) + @config_service.clear_token_in_config(config_file_name) + return "Your access token looked invalid, so we've reset it to be blank. Please re-run rails grade and copy-paste your token carefully from the assignment page." + end + + # Sync specs if needed + if GradeRunner.override_local_specs + resource_info = @token_service.fetch_upstream_repo(token) + full_reponame = resource_info.fetch("repo_slug") + remote_spec_folder_sha = resource_info.fetch("spec_folder_sha") + source_code_url = resource_info.fetch("source_code_url") + + @github_service.set_upstream_remote(full_reponame) + # @spec_service.sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) + end + + # Run tests + output_path = @spec_service.prepare_output_directory + rspec_output_json = @spec_service.run_tests(output_path) + + # Submit results + username = github_username + reponame = @github_service.get_repo_name + sha = @github_service.get_commit_sha + + GradeRunner::Runner.new(@submission_url, token, rspec_output_json, username, reponame, sha, "manual").process + + "Grade submitted successfully" + end + + def process_reset_token + config_file_name = @config_service.get_config_file_path + github_username = @github_service.retrieve_github_username(config_file_name) + + # Get new token from user + token = @token_service.prompt_for_token(config_file_name) + + # Save token + @config_service.save_token_to_config(config_file_name, token, @submission_url, github_username) + + "Grade token has been reset successfully." + end + end + end +end \ No newline at end of file diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb new file mode 100644 index 0000000..2222eaf --- /dev/null +++ b/lib/grade_runner/services/spec_service.rb @@ -0,0 +1,33 @@ +require "oj" +require "grade_runner/utils/path_utils" + +module GradeRunner + module Services + class SpecService + def initialize + @path_utils = GradeRunner::Utils::PathUtils + end + + def run_tests(output_path) + # Ensure database is migrated if Rails app + `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) + + # Run tests with JSON formatter + `RAILS_ENV=test bundle exec rspec --format JsonOutputFormatter --out #{output_path}` + + # Load and return test results + Oj.load(File.read(output_path)) + end + + def sync_specs_with_source(full_reponame, remote_sha, repo_url) + # Implement spec synchronization logic here + # Currently commented out in the original + # This is a placeholder for future implementation + end + + def prepare_output_directory + @path_utils.tmp_output_path + end + end + end +end \ No newline at end of file diff --git a/lib/grade_runner/services/token_service.rb b/lib/grade_runner/services/token_service.rb new file mode 100644 index 0000000..8952ac0 --- /dev/null +++ b/lib/grade_runner/services/token_service.rb @@ -0,0 +1,70 @@ +require "net/http" +require "oj" + +module GradeRunner + module Services + class TokenService + attr_reader :submission_url + + def initialize(submission_url) + @submission_url = submission_url + end + + def validate_token(token) + return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ + + url = "#{submission_url}/submissions/validate_token?token=#{token}" + uri = URI.parse(url) + req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') + res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| + http.request(req) + end + result = Oj.load(res.body) + result["success"] + rescue => e + false + end + + def get_token(input_token, file_token, config_file_name) + if input_token.present? + input_token + elsif file_token.present? + file_token + else + prompt_for_token(config_file_name) + end + end + + def prompt_for_token(config_file_name) + puts "Enter your access token for this project" + new_token = "" + + while new_token.empty? do + print "> " + new_token = $stdin.gets.chomp.strip + + if new_token.empty? || !validate_token(new_token) + puts "Please enter valid token" + new_token = "" + end + end + + new_token + end + + def fetch_upstream_repo(token) + return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ + + url = "#{submission_url}/submissions/resource?token=#{token}" + uri = URI.parse(url) + req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') + res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| + http.request(req) + end + Oj.load(res.body) + rescue => e + false + end + end + end +end \ No newline at end of file diff --git a/lib/grade_runner/utils/path_utils.rb b/lib/grade_runner/utils/path_utils.rb new file mode 100644 index 0000000..23813e4 --- /dev/null +++ b/lib/grade_runner/utils/path_utils.rb @@ -0,0 +1,42 @@ +module GradeRunner + module Utils + class PathUtils + class << self + # Determine the project root directory + # @return [Pathname] The project root path + def project_root + if defined?(Rails) + Rails.root + elsif defined?(Bundler) + Bundler.root + else + Pathname.new(Dir.pwd) + end + end + + # Create a path relative to project root + # @param path [String] Relative path + # @return [Pathname] Absolute path from project root + def path_in_project(path) + project_root.join(path) + end + + # Find or create a directory in the project + # @param directory_name [String] Directory name to find or create + # @return [String] Path to the directory + def find_or_create_directory(directory_name) + directory = path_in_project(directory_name) + FileUtils.mkdir_p(directory) unless Dir.exist?(directory) + directory.to_s + end + + # Get the temporary output directory path for test results + # @return [String] Path to the output directory + def tmp_output_path + output_dir = find_or_create_directory("tmp/output") + File.join(output_dir, "#{Time.now.to_i}.json") + end + end + end + end +end \ No newline at end of file diff --git a/lib/tasks/grade.rake b/lib/tasks/grade.rake index f236fdd..98cdf66 100644 --- a/lib/tasks/grade.rake +++ b/lib/tasks/grade.rake @@ -1,12 +1,9 @@ require "active_support/core_ext/object/blank" require "grade_runner/runner" -require "octokit" -require "yaml" -require "zip" -require "fileutils" -require "open-uri" +require "grade_runner/services/grade_service" +require "grade_runner/utils/path_utils" -desc "Alias for \"grade:next\"." +desc "Alias for \"grade:all\"." task grade: "grade:all" do end @@ -15,263 +12,18 @@ namespace :grade do task :all do ARGV.each { |a| task a.to_sym do ; end } input_token = ARGV[1] - file_token = nil - config_dir_name = find_or_create_directory(".vscode") - config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - student_config = {} - student_config["submission_url"] = "https://f5cd-98-227-60-153.ngrok-free.app" - student_config["github_username"] = retrieve_github_username + grade_service = GradeRunner::Services::GradeService.new + result = grade_service.process_grade_all(input_token) - if File.exist?(config_file_name) - begin - config = YAML.load_file(config_file_name) - rescue - abort "It looks like there's something wrong with your token in `#{config_dir_name}/.ltici_apitoken.yml`. Please delete that file and try `rails grade` again, and be sure to provide the access token for THIS project.".red - end - submission_url = config["submission_url"] - file_token = config["personal_access_token"] - student_config["submission_url"] = config["submission_url"] - else - submission_url = "https://f5cd-98-227-60-153.ngrok-free.app" - end - if file_token.nil? && ENV.has_key?("LTICI_GITPOD_APITOKEN") - input_token = ENV.fetch("LTICI_GITPOD_APITOKEN") - end - if input_token.present? - token = input_token - student_config["personal_access_token"] = input_token - update_config_file(config_file_name, student_config) - elsif input_token.nil? && file_token.present? - token = file_token - elsif input_token.nil? && file_token.nil? - puts "Enter your access token for this project" - new_personal_access_token = "" - - while new_personal_access_token == "" do - print "> " - new_personal_access_token = $stdin.gets.chomp.strip - if new_personal_access_token!= "" && is_valid_token?(submission_url, new_personal_access_token) == false - puts "Please enter valid token" - new_personal_access_token = "" - end - - if new_personal_access_token != "" - student_config["personal_access_token"] = new_personal_access_token - update_config_file(config_file_name, student_config) - token = new_personal_access_token - end - end - end - - if token.present? - - if is_valid_token?(submission_url, token) == false - student_config["personal_access_token"] = nil - update_config_file(config_file_name, student_config) - puts "Your access token looked invalid, so we've reset it to be blank. Please re-run rails grade and, when asked, copy-paste your token carefully from the assignment page." - else - if GradeRunner.override_local_specs - resource_info = upstream_repo(submission_url, token) - full_reponame = resource_info.fetch("repo_slug") - remote_spec_folder_sha = resource_info.fetch("spec_folder_sha") - source_code_url = resource_info.fetch("source_code_url") - set_upstream_remote(full_reponame) - # TODO: verify specs match source code - # sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) - end - - path = File.join(project_root, "/tmp/output/#{Time.now.to_i}.json") - `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - `RAILS_ENV=test bundle exec rspec --format JsonOutputFormatter --out #{path}` - rspec_output_json = Oj.load(File.read(path)) - username = retrieve_github_username - reponame = project_root.to_s.split("/").last - sha = `git rev-parse HEAD`.slice(0..7) - - GradeRunner::Runner.new(submission_url, token, rspec_output_json, username, reponame, sha, "manual").process - end - else - puts "We couldn't find your access token, so we couldn't record your grade. Please click on the assignment link again and run the rails grade ... command shown there." - end + puts result if result.present? end - # desc "Run only the next failing test." - # task :next do - # path = File.join(project_root, "examples.txt") - # if File.exist?(path) - # `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - # puts `RAILS_ENV=test bundle exec rspec --next-failure --format HintFormatter` - # else - # puts `RAILS_ENV=test bundle exec rspec` - # puts "Please rerun rails grade:next to run the first failing spec" - # end - # end - desc "Reset access token saved in YAML file." task :reset_token do - config_dir_name = find_or_create_directory(".vscode") - config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - submission_url = "https://f5cd-98-227-60-153.ngrok-free.app" - - student_config = {} - student_config["submission_url"] = submission_url - student_config["github_username"] = retrieve_github_username - puts "Enter your access token for this project" - new_personal_access_token = "" - - while new_personal_access_token == "" do - print "> " - new_personal_access_token = $stdin.gets.chomp.strip - token_valid = is_valid_token?(submission_url, new_personal_access_token) - unless token_valid && !new_personal_access_token.empty? - puts "Please enter valid token" - new_personal_access_token = "" - end - - unless new_personal_access_token.empty? - student_config["personal_access_token"] = new_personal_access_token - update_config_file(config_file_name, student_config) - token = new_personal_access_token - end - end - puts "Grade token has been reset successfully." - end - -end - -# def sync_specs_with_source(full_reponame, remote_sha, repo_url) -# # Unstage staged changes in spec folder -# `git restore --staged spec/* ` -# # Discard unstaged changes in spec folder -# `git checkout spec -q` -# `git clean spec -f -q` -# local_sha = `git ls-tree HEAD #{project_root.join('spec')}`.chomp.split[2] - -# unless remote_sha == local_sha -# find_or_create_directory("tmp") -# find_or_create_directory("tmp/backup") -# files_and_subfolders_inside_specs = Dir.glob("spec/*") -# # Temporarily move specs -# FileUtils.mv(files_and_subfolders_inside_specs, "tmp/backup") - -# download_file(repo_url, "tmp/spec.zip") -# extracted_zip_folder = extract_zip("tmp/spec.zip", "tmp") -# source_directory = extracted_zip_folder.join("spec") -# overwrite_spec_folder(source_directory) - -# FileUtils.rm(project_root.join("tmp/spec.zip")) -# FileUtils.rm_rf(extracted_zip_folder) -# FileUtils.rm_rf("tmp/backup") -# `git add spec/` -# `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft "` -# end -# end - -# def download_file(url, destination) -# download = URI.open(url) -# IO.copy_stream(download, destination) -# end - -# def extract_zip(folder, destination) -# extracted_file_path = project_root.join(destination) -# Zip::File.open(folder) do |zip_file| -# zip_file.each_with_index do |file, index| -# # Get name of root folder in zip file -# if index == 0 -# extracted_file_path = extracted_file_path.join(file.name) -# end -# file_path = File.join(destination, file.name) -# FileUtils.mkdir_p(File.dirname(file_path)) -# file.extract(file_path) -# end -# end -# extracted_file_path -# end - -# def overwrite_spec_folder(source_directory) -# destination_directory = "spec" -# # Get all files in the source directory -# files = Dir.glob("#{source_directory}/*") -# # Move each file to the destination directory -# files.each do |file| -# FileUtils.mv(file, destination_directory) -# end -# end - -def set_upstream_remote(repo_slug) - upstream = `git remote -v | grep -w upstream`.chomp - if upstream.blank? - `git remote add upstream https://github.com/#{repo_slug}` - else - `git remote set-url upstream https://github.com/#{repo_slug}` - end -end - -def update_config_file(config_file_name, config) - File.write(config_file_name, YAML.dump(config)) -end - -def find_or_create_directory(directory_name) - directory = File.join(project_root, directory_name) - Dir.mkdir(directory) unless Dir.exist?(directory) - directory -end - -def is_valid_token?(root_url, token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ - url = "#{root_url}/submissions/validate_token?token=#{token}" - uri = URI.parse(url) - req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') - res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| - http.request(req) - end - result = Oj.load(res.body) - result["success"] -rescue => e - return false -end - -def upstream_repo(root_url, token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ - url = "#{root_url}/submissions/resource?token=#{token}" - uri = URI.parse(url) - req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') - res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| - http.request(req) - end - Oj.load(res.body) -rescue => e - return false -end - -def retrieve_github_username - config_dir_name = find_or_create_directory(".vscode") - config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - if File.exist?(config_file_name) - config = YAML.load_file(config_file_name) - if config["github_username"].present? - return config["github_username"] - end - else - github_email = `git config user.email`.chomp - return "" if github_email.blank? - username = `git config user.name`.chomp - search_results = Octokit.search_users("#{github_email} in:email").fetch(:items) - if search_results.present? - username = search_results.first.fetch(:login, username) - end - return username - end -end - -def project_root - if defined?(Rails) - return Rails.root - end + grade_service = GradeRunner::Services::GradeService.new + result = grade_service.process_reset_token - if defined?(Bundler) - return Bundler.root + puts result if result.present? end - Dir.pwd end diff --git a/lib/tasks/grade_runner.rake b/lib/tasks/grade_runner.rake index 6ade520..a7e84eb 100644 --- a/lib/tasks/grade_runner.rake +++ b/lib/tasks/grade_runner.rake @@ -1,25 +1,22 @@ +require "grade_runner/services/grade_service" +require "grade_runner/utils/path_utils" + namespace :grade_runner do desc "Grade project" task :runner do - default_submission_url = "https://f5cd-98-227-60-153.ngrok-free.app" - config = {} - path = Rails.root.join("grades.yml") - if File.exist?(path) - begin - config = YAML.load_file(path) - rescue - abort "It looks like there's something wrong with your token in `/grades.yml`. Please delete that file and try `rails grade:all` again, and be sure to provide the access token for THIS project.".red - end - end + # Handle CI-specific paths and environment rspec_output_json = JSON.parse(File.read("#{ENV['CIRCLE_ARTIFACTS']}/output/rspec_output.json")) username = ENV["CIRCLE_PROJECT_USERNAME"] reponame = ENV["CIRCLE_PROJECT_REPONAME"] sha = ENV["CIRCLE_SHA1"] token = ENV['GRADES_PERSONAL_ACCESS_TOKEN'] + if token.present? - GradeRunner::Runner.new('', config['submission_url'] || default_submission_url, token, rspec_output_json, username, reponame, sha, 'circle_ci').process + # If in CI environment, use Runner directly with CI parameters + submission_url = GradeRunner.submission_url + GradeRunner::Runner.new(submission_url, token, rspec_output_json, username, reponame, sha, 'circle_ci').process else puts "We couldn't find your access token, so we couldn't record your grade. Please click on the assignment link again and run the rails grade ... command shown there." end end -end +end \ No newline at end of file From a1eed91aa39de93f73a78cf17dae39f64128e382 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 09:20:44 -0500 Subject: [PATCH 05/34] Remove extra config line in readme --- README.markdown | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.markdown b/README.markdown index 30ca05a..959d9d3 100644 --- a/README.markdown +++ b/README.markdown @@ -92,12 +92,6 @@ require "rubygems" require "bundler/setup" require "rake" -# Configure grade_runner (optional) -require "grade_runner" -GradeRunner.config do |config| - config.submission_url = "https://your-grading-server-url.com" # Set custom submission URL -end - dir = Gem::Specification.find_by_name("grade_runner").gem_dir load "#{dir}/lib/tasks/grade.rake" From c8de4837379dc2d096dbf994438ea8119a00ae67 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 09:46:07 -0500 Subject: [PATCH 06/34] Implement spec_service so we can still override local specs if needed. --- lib/grade_runner/services/grade_service.rb | 34 +++++------ lib/grade_runner/services/spec_service.rb | 34 ++++++++++- lib/grade_runner/utils/path_utils.rb | 6 ++ spec/grade_service_spec.rb | 70 ++++++++++++++++++++++ spec/spec_service_spec.rb | 23 +++++++ 5 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 spec/grade_service_spec.rb create mode 100644 spec/spec_service_spec.rb diff --git a/lib/grade_runner/services/grade_service.rb b/lib/grade_runner/services/grade_service.rb index 4a21cd8..adc1854 100644 --- a/lib/grade_runner/services/grade_service.rb +++ b/lib/grade_runner/services/grade_service.rb @@ -12,68 +12,68 @@ def initialize def process_grade_all(input_token = nil) config_file_name = @config_service.get_config_file_path config = @config_service.load_config(config_file_name, @submission_url) - + file_token = config["personal_access_token"] - + # Check for Gitpod environment token if file_token.nil? && ENV.has_key?("LTICI_GITPOD_APITOKEN") input_token = ENV.fetch("LTICI_GITPOD_APITOKEN") end - + # Get token (from input, file, or prompt) token = @token_service.get_token(input_token, file_token, config_file_name) github_username = @github_service.retrieve_github_username(config_file_name) - + # Save token to config if new if input_token.present? || (token.present? && file_token.nil?) @config_service.save_token_to_config(config_file_name, token, @submission_url, github_username) end - + return "We couldn't find your access token. Please click on the assignment link and run the rails grade command shown there." if token.blank? - + # Validate token if !@token_service.validate_token(token) @config_service.clear_token_in_config(config_file_name) return "Your access token looked invalid, so we've reset it to be blank. Please re-run rails grade and copy-paste your token carefully from the assignment page." end - + # Sync specs if needed if GradeRunner.override_local_specs resource_info = @token_service.fetch_upstream_repo(token) full_reponame = resource_info.fetch("repo_slug") remote_spec_folder_sha = resource_info.fetch("spec_folder_sha") source_code_url = resource_info.fetch("source_code_url") - + @github_service.set_upstream_remote(full_reponame) - # @spec_service.sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) + @spec_service.sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) end - + # Run tests output_path = @spec_service.prepare_output_directory rspec_output_json = @spec_service.run_tests(output_path) - + # Submit results username = github_username reponame = @github_service.get_repo_name sha = @github_service.get_commit_sha - + GradeRunner::Runner.new(@submission_url, token, rspec_output_json, username, reponame, sha, "manual").process - + "Grade submitted successfully" end def process_reset_token config_file_name = @config_service.get_config_file_path github_username = @github_service.retrieve_github_username(config_file_name) - + # Get new token from user token = @token_service.prompt_for_token(config_file_name) - + # Save token @config_service.save_token_to_config(config_file_name, token, @submission_url, github_username) - + "Grade token has been reset successfully." end end end -end \ No newline at end of file +end diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb index 2222eaf..a180ce4 100644 --- a/lib/grade_runner/services/spec_service.rb +++ b/lib/grade_runner/services/spec_service.rb @@ -1,4 +1,5 @@ require "oj" +require "fileutils" require "grade_runner/utils/path_utils" module GradeRunner @@ -20,9 +21,36 @@ def run_tests(output_path) end def sync_specs_with_source(full_reponame, remote_sha, repo_url) - # Implement spec synchronization logic here - # Currently commented out in the original - # This is a placeholder for future implementation + # Return if required parameters are missing + return false unless full_reponame && remote_sha && repo_url + + # Create a temporary directory for clone + temp_dir = "#{@path_utils.tmp_path}/upstream_repo" + FileUtils.rm_rf(temp_dir) if Dir.exist?(temp_dir) + FileUtils.mkdir_p(temp_dir) + + Dir.chdir(temp_dir) do + # Clone the upstream repository + `git clone https://github.com/#{full_reponame} .` + + # Checkout the specific SHA if provided + `git checkout #{remote_sha}` if remote_sha.present? + + # Copy spec directory if it exists + if Dir.exist?("spec") + # Remove existing specs in project + FileUtils.rm_rf("#{@path_utils.project_root}/spec") + + # Copy specs from upstream to project + FileUtils.cp_r("spec", "#{@path_utils.project_root}/") + return true + end + end + + false + ensure + # Clean up temporary directory + FileUtils.rm_rf(temp_dir) if Dir.exist?(temp_dir) end def prepare_output_directory diff --git a/lib/grade_runner/utils/path_utils.rb b/lib/grade_runner/utils/path_utils.rb index 23813e4..a229985 100644 --- a/lib/grade_runner/utils/path_utils.rb +++ b/lib/grade_runner/utils/path_utils.rb @@ -36,6 +36,12 @@ def tmp_output_path output_dir = find_or_create_directory("tmp/output") File.join(output_dir, "#{Time.now.to_i}.json") end + + # Get the temporary directory path + # @return [String] Path to the temporary directory + def tmp_path + find_or_create_directory("tmp") + end end end end diff --git a/spec/grade_service_spec.rb b/spec/grade_service_spec.rb new file mode 100644 index 0000000..ca99c00 --- /dev/null +++ b/spec/grade_service_spec.rb @@ -0,0 +1,70 @@ +require 'spec_helper' + +describe GradeRunner::Services::GradeService do + let(:grade_service) { GradeRunner::Services::GradeService.new } + + before do + # Save original value + @original_override_value = GradeRunner.override_local_specs + + # Mock dependencies + allow_any_instance_of(GradeRunner::Services::ConfigService).to receive(:get_config_file_path).and_return('config_file.yml') + allow_any_instance_of(GradeRunner::Services::ConfigService).to receive(:load_config).and_return({ 'personal_access_token' => 'token' }) + allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:get_token).and_return('valid_token') + allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:validate_token).and_return(true) + allow_any_instance_of(GradeRunner::Services::GithubService).to receive(:retrieve_github_username).and_return('username') + end + + after do + # Restore original value + GradeRunner.override_local_specs = @original_override_value + end + + describe '#process_grade_all' do + context 'when specs synchronization is enabled' do + it 'calls sync_specs_with_source when override_local_specs is true' do + # Set up configuration + GradeRunner.override_local_specs = true + + # Mock token fetch to avoid API calls + resource_info = { + 'repo_slug' => 'org/repo', + 'spec_folder_sha' => 'sha123', + 'source_code_url' => 'url' + } + + allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:fetch_upstream_repo).and_return(resource_info) + + # Expectations + expect_any_instance_of(GradeRunner::Services::GithubService).to receive(:set_upstream_remote).with('org/repo') + expect_any_instance_of(GradeRunner::Services::SpecService).to receive(:sync_specs_with_source).with('org/repo', 'sha123', 'url') + + # Skip the actual test running and submission + allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:prepare_output_directory).and_return('output.json') + allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:run_tests).and_return({}) + allow_any_instance_of(GradeRunner::Runner).to receive(:process).and_return(true) + + grade_service.process_grade_all + end + end + + context 'when specs synchronization is disabled' do + it 'does not call sync_specs_with_source when override_local_specs is false' do + # Set up configuration + GradeRunner.override_local_specs = false + + # Expectations + expect_any_instance_of(GradeRunner::Services::TokenService).not_to receive(:fetch_upstream_repo) + expect_any_instance_of(GradeRunner::Services::GithubService).not_to receive(:set_upstream_remote) + expect_any_instance_of(GradeRunner::Services::SpecService).not_to receive(:sync_specs_with_source) + + # Skip the actual test running and submission + allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:prepare_output_directory).and_return('output.json') + allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:run_tests).and_return({}) + allow_any_instance_of(GradeRunner::Runner).to receive(:process).and_return(true) + + grade_service.process_grade_all + end + end + end +end \ No newline at end of file diff --git a/spec/spec_service_spec.rb b/spec/spec_service_spec.rb new file mode 100644 index 0000000..e80f154 --- /dev/null +++ b/spec/spec_service_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' +require 'fileutils' +require 'tempfile' +require 'tmpdir' + +describe GradeRunner::Services::SpecService do + let(:spec_service) { GradeRunner::Services::SpecService.new } + + describe '#sync_specs_with_source' do + it 'returns false when required parameters are missing' do + expect(spec_service.sync_specs_with_source(nil, 'sha123', 'url')).to be false + expect(spec_service.sync_specs_with_source('org/repo', nil, 'url')).to be false + expect(spec_service.sync_specs_with_source('org/repo', 'sha123', nil)).to be false + end + + # This test would need to mock Git operations to fully test the functionality + it 'handles git operations properly' do + # Mock to ensure we're not actually trying to clone a repo or run git commands + expect(Dir).not_to receive(:chdir) + expect(spec_service.sync_specs_with_source('org/repo', 'sha123', 'url')).to be false + end + end +end \ No newline at end of file From 056579fe32202e660e26c1a9155c9c552c0939ef Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 11:40:39 -0500 Subject: [PATCH 07/34] Upgrade bundler because DidYouMean::SPELL_CHECKERS is deprecated. Add explicit ruby version 3.2 because newer versions (eg 3.4) are not supported (eg BigDecimal). --- Gemfile | 12 +++++++----- Gemfile.lock | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index dc2b96c..ad44698 100644 --- a/Gemfile +++ b/Gemfile @@ -1,18 +1,20 @@ source "https://rubygems.org" -# Add dependencies required to use your gem here. -# Example: + +# TODO: will this break projects? can we set a range? +# 3.4 does not work at the moment +ruby "3.2.1" + gem "activesupport", ">= 2.3.5" gem "oj", "~> 3.13.12" gem "octokit", "~> 5.0" gem "zip" gem "faraday-retry", "~> 1.0.3" gem "rake", "~> 13" -# Add dependencies to develop your gem here. -# Include everything needed to run rake, tests, features, etc. + group :development do gem "rspec", "~> 3.5.0" gem "rdoc", "~> 6.1" - gem "bundler", "~> 2.1.4" + gem "bundler", "~> 2.6.7" gem "juwelier", "~> 2.1.0" gem "simplecov", ">= 0" gem "pry", "~> 0" diff --git a/Gemfile.lock b/Gemfile.lock index e3877ff..61f9f3f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -152,7 +152,7 @@ PLATFORMS DEPENDENCIES activesupport (>= 2.3.5) - bundler (~> 2.1.4) + bundler (~> 2.6.7) faraday-retry (~> 1.0.3) juwelier (~> 2.1.0) octokit (~> 5.0) @@ -170,4 +170,4 @@ DEPENDENCIES zip BUNDLED WITH - 2.1.4 + 2.6.7 From 9fc01e9a6f195771035e4e0b6e5b1af7dd045b60 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 11:45:40 -0500 Subject: [PATCH 08/34] Require active support gem so we can call present? and update spec service tests. --- lib/grade_runner/services/grade_service.rb | 2 ++ lib/grade_runner/services/spec_service.rb | 7 ++++--- lib/grade_runner/services/token_service.rb | 1 + spec/spec_service_spec.rb | 16 ++++++++++++++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/grade_runner/services/grade_service.rb b/lib/grade_runner/services/grade_service.rb index adc1854..30b68f2 100644 --- a/lib/grade_runner/services/grade_service.rb +++ b/lib/grade_runner/services/grade_service.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/object/blank" + module GradeRunner module Services class GradeService diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb index a180ce4..5bf580d 100644 --- a/lib/grade_runner/services/spec_service.rb +++ b/lib/grade_runner/services/spec_service.rb @@ -1,6 +1,7 @@ require "oj" require "fileutils" require "grade_runner/utils/path_utils" +require "active_support/core_ext/object/blank" module GradeRunner module Services @@ -25,8 +26,8 @@ def sync_specs_with_source(full_reponame, remote_sha, repo_url) return false unless full_reponame && remote_sha && repo_url # Create a temporary directory for clone - temp_dir = "#{@path_utils.tmp_path}/upstream_repo" - FileUtils.rm_rf(temp_dir) if Dir.exist?(temp_dir) + temp_dir = File.join(@path_utils.tmp_path, "upstream_repo") + FileUtils.rm_rf(temp_dir) if temp_dir && Dir.exist?(temp_dir) FileUtils.mkdir_p(temp_dir) Dir.chdir(temp_dir) do @@ -50,7 +51,7 @@ def sync_specs_with_source(full_reponame, remote_sha, repo_url) false ensure # Clean up temporary directory - FileUtils.rm_rf(temp_dir) if Dir.exist?(temp_dir) + FileUtils.rm_rf(temp_dir) if temp_dir && Dir.exist?(temp_dir) end def prepare_output_directory diff --git a/lib/grade_runner/services/token_service.rb b/lib/grade_runner/services/token_service.rb index 8952ac0..c7e87f6 100644 --- a/lib/grade_runner/services/token_service.rb +++ b/lib/grade_runner/services/token_service.rb @@ -1,5 +1,6 @@ require "net/http" require "oj" +require "active_support/core_ext/object/blank" module GradeRunner module Services diff --git a/spec/spec_service_spec.rb b/spec/spec_service_spec.rb index e80f154..6b67f85 100644 --- a/spec/spec_service_spec.rb +++ b/spec/spec_service_spec.rb @@ -7,6 +7,15 @@ let(:spec_service) { GradeRunner::Services::SpecService.new } describe '#sync_specs_with_source' do + before do + # Mock the path utils + allow(GradeRunner::Utils::PathUtils).to receive(:tmp_path).and_return('/tmp/mock_path') + allow(FileUtils).to receive(:rm_rf) + allow(FileUtils).to receive(:mkdir_p) + allow(Dir).to receive(:chdir).and_yield + allow(Dir).to receive(:exist?).and_return(false) + end + it 'returns false when required parameters are missing' do expect(spec_service.sync_specs_with_source(nil, 'sha123', 'url')).to be false expect(spec_service.sync_specs_with_source('org/repo', nil, 'url')).to be false @@ -15,8 +24,11 @@ # This test would need to mock Git operations to fully test the functionality it 'handles git operations properly' do - # Mock to ensure we're not actually trying to clone a repo or run git commands - expect(Dir).not_to receive(:chdir) + # Setup git mocks + expect(spec_service).to receive(:`).with('git clone https://github.com/org/repo .').and_return('') + expect(spec_service).to receive(:`).with('git checkout sha123').and_return('') + + # Test the method expect(spec_service.sync_specs_with_source('org/repo', 'sha123', 'url')).to be false end end From 999acbabbd25d8201efcc3f68adf031a96e99d16 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 11:48:02 -0500 Subject: [PATCH 09/34] Update note on Gemfile --- Gemfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index ad44698..9176fd9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source "https://rubygems.org" -# TODO: will this break projects? can we set a range? -# 3.4 does not work at the moment +# TODO: will this break projects on different ruby versions? can we set a range? +# Note: grade_runner does not support 3.4 (BigDecimal) ruby "3.2.1" gem "activesupport", ">= 2.3.5" From c17c77089f8c3f030cb0b1f309b1fdc900f7b089 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 11:49:56 -0500 Subject: [PATCH 10/34] Increment version since this is a significant refactor --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 43b2961..9789c4c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.13 +0.0.14 From 13fa918d192b2a6ae6a82bd515a0ea727f8dafd5 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 11:52:37 -0500 Subject: [PATCH 11/34] Refactor token regex to a constant --- lib/grade_runner/services/token_service.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/grade_runner/services/token_service.rb b/lib/grade_runner/services/token_service.rb index c7e87f6..8a91f20 100644 --- a/lib/grade_runner/services/token_service.rb +++ b/lib/grade_runner/services/token_service.rb @@ -7,13 +7,16 @@ module Services class TokenService attr_reader :submission_url + # TODO: do we want to make this configurable? + TOKEN_REGEX = /^[1-9A-Za-z][^OIl]{23}$/ + def initialize(submission_url) @submission_url = submission_url end def validate_token(token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ - + return false unless token.is_a?(String) && token =~ TOKEN_REGEX + url = "#{submission_url}/submissions/validate_token?token=#{token}" uri = URI.parse(url) req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') @@ -43,19 +46,19 @@ def prompt_for_token(config_file_name) while new_token.empty? do print "> " new_token = $stdin.gets.chomp.strip - + if new_token.empty? || !validate_token(new_token) puts "Please enter valid token" new_token = "" end end - + new_token end def fetch_upstream_repo(token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ - + return false unless token.is_a?(String) && token =~ TOKEN_REGEX + url = "#{submission_url}/submissions/resource?token=#{token}" uri = URI.parse(url) req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') @@ -68,4 +71,4 @@ def fetch_upstream_repo(token) end end end -end \ No newline at end of file +end From 3fcfcbac60ccbb4a8e217e2817f2231caf0580bd Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 16:06:08 -0500 Subject: [PATCH 12/34] Remove unused requires. --- lib/tasks/grade.rake | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/tasks/grade.rake b/lib/tasks/grade.rake index 98cdf66..5a60a1d 100644 --- a/lib/tasks/grade.rake +++ b/lib/tasks/grade.rake @@ -1,7 +1,5 @@ require "active_support/core_ext/object/blank" -require "grade_runner/runner" require "grade_runner/services/grade_service" -require "grade_runner/utils/path_utils" desc "Alias for \"grade:all\"." task grade: "grade:all" do From 72412962742bdaf1bbd6c67fbcab796412a9e80b Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 16:29:16 -0500 Subject: [PATCH 13/34] Add back zip download function to spec service. --- lib/grade_runner/services/spec_service.rb | 113 +++++++++++++++++----- spec/spec_service_spec.rb | 26 +++-- 2 files changed, 109 insertions(+), 30 deletions(-) diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb index 5bf580d..63f1b04 100644 --- a/lib/grade_runner/services/spec_service.rb +++ b/lib/grade_runner/services/spec_service.rb @@ -2,6 +2,8 @@ require "fileutils" require "grade_runner/utils/path_utils" require "active_support/core_ext/object/blank" +require "open-uri" +require "zip" module GradeRunner module Services @@ -24,34 +26,97 @@ def run_tests(output_path) def sync_specs_with_source(full_reponame, remote_sha, repo_url) # Return if required parameters are missing return false unless full_reponame && remote_sha && repo_url - - # Create a temporary directory for clone - temp_dir = File.join(@path_utils.tmp_path, "upstream_repo") - FileUtils.rm_rf(temp_dir) if temp_dir && Dir.exist?(temp_dir) - FileUtils.mkdir_p(temp_dir) - - Dir.chdir(temp_dir) do - # Clone the upstream repository - `git clone https://github.com/#{full_reponame} .` + + # Unstage staged changes in spec folder + `git restore --staged spec/* 2>/dev/null` + # Discard unstaged changes in spec folder + `git checkout spec -q 2>/dev/null` + `git clean spec -f -q 2>/dev/null` + + # Get local SHA of spec folder + local_sha_output = `git ls-tree HEAD #{@path_utils.project_root}/spec 2>/dev/null`.chomp + local_sha = local_sha_output.split[2] if local_sha_output.present? + + # Only update if remote SHA differs from local SHA + unless remote_sha == local_sha + # Create temporary directories + tmp_dir = @path_utils.find_or_create_directory("tmp") + backup_dir = @path_utils.find_or_create_directory("tmp/backup") + + # Backup existing specs if they exist + if Dir.exist?("#{@path_utils.project_root}/spec") + files_and_subfolders = Dir.glob("#{@path_utils.project_root}/spec/*") + FileUtils.mv(files_and_subfolders, backup_dir) if files_and_subfolders.any? + else + FileUtils.mkdir_p("#{@path_utils.project_root}/spec") + end + + # Download spec zip file + zip_path = "#{tmp_dir}/spec.zip" + download_file(repo_url, zip_path) + + # Extract zip file + extracted_folder = extract_zip(zip_path, tmp_dir) + source_directory = File.join(extracted_folder, "spec") + + # Overwrite spec folder with new specs + overwrite_spec_folder(source_directory) + + # Clean up + FileUtils.rm(zip_path) if File.exist?(zip_path) + FileUtils.rm_rf(extracted_folder) if extracted_folder && Dir.exist?(extracted_folder) + FileUtils.rm_rf(backup_dir) if Dir.exist?(backup_dir) + + # Commit changes + `git add spec/ 2>/dev/null` + `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft " 2>/dev/null` + + return true + end + + false + rescue => e + # In case of error, still try to clean up + tmp_zip = "#{@path_utils.tmp_path}/spec.zip" + FileUtils.rm(tmp_zip) if File.exist?(tmp_zip) + FileUtils.rm_rf("#{@path_utils.tmp_path}/backup") if Dir.exist?("#{@path_utils.tmp_path}/backup") + false + end - # Checkout the specific SHA if provided - `git checkout #{remote_sha}` if remote_sha.present? + def download_file(url, destination) + download = URI.open(url) + IO.copy_stream(download, destination) + end - # Copy spec directory if it exists - if Dir.exist?("spec") - # Remove existing specs in project - FileUtils.rm_rf("#{@path_utils.project_root}/spec") - - # Copy specs from upstream to project - FileUtils.cp_r("spec", "#{@path_utils.project_root}/") - return true + def extract_zip(folder, destination) + extracted_file_path = destination + + Zip::File.open(folder) do |zip_file| + zip_file.each_with_index do |file, index| + # Get name of root folder in zip file + if index == 0 + extracted_file_path = File.join(destination, file.name) + end + + file_path = File.join(destination, file.name) + FileUtils.mkdir_p(File.dirname(file_path)) + file.extract(file_path) unless File.exist?(file_path) end end - - false - ensure - # Clean up temporary directory - FileUtils.rm_rf(temp_dir) if temp_dir && Dir.exist?(temp_dir) + + extracted_file_path + end + + def overwrite_spec_folder(source_directory) + destination_directory = "#{@path_utils.project_root}/spec" + + # Get all files in the source directory + files = Dir.glob("#{source_directory}/*") + + # Move each file to the destination directory + files.each do |file| + FileUtils.cp_r(file, destination_directory) + end end def prepare_output_directory diff --git a/spec/spec_service_spec.rb b/spec/spec_service_spec.rb index 6b67f85..fa5097d 100644 --- a/spec/spec_service_spec.rb +++ b/spec/spec_service_spec.rb @@ -10,10 +10,17 @@ before do # Mock the path utils allow(GradeRunner::Utils::PathUtils).to receive(:tmp_path).and_return('/tmp/mock_path') + allow(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).and_return('/tmp/mock_path') + allow(GradeRunner::Utils::PathUtils).to receive(:project_root).and_return('/tmp/mock_project') allow(FileUtils).to receive(:rm_rf) allow(FileUtils).to receive(:mkdir_p) - allow(Dir).to receive(:chdir).and_yield + allow(FileUtils).to receive(:mv) + allow(FileUtils).to receive(:cp_r) + allow(FileUtils).to receive(:rm) allow(Dir).to receive(:exist?).and_return(false) + allow(Dir).to receive(:glob).and_return([]) + allow(File).to receive(:exist?).and_return(false) + allow(File).to receive(:join) { |*args| args.join('/') } end it 'returns false when required parameters are missing' do @@ -22,14 +29,21 @@ expect(spec_service.sync_specs_with_source('org/repo', 'sha123', nil)).to be false end - # This test would need to mock Git operations to fully test the functionality + # Test for git operations and spec syncing it 'handles git operations properly' do - # Setup git mocks - expect(spec_service).to receive(:`).with('git clone https://github.com/org/repo .').and_return('') - expect(spec_service).to receive(:`).with('git checkout sha123').and_return('') + # Mock all the git shell commands + allow(spec_service).to receive(:`).with(any_args).and_return('') + + # Mock URI.open and zip extraction + allow(spec_service).to receive(:download_file).and_return(true) + allow(spec_service).to receive(:extract_zip).and_return('/tmp/mock_path/extracted') + allow(spec_service).to receive(:overwrite_spec_folder).and_return(true) + + # Force the method to update specs by making SHA comparison fail + allow(spec_service).to receive(:`).with(/git ls-tree/).and_return("blob 100644 different-sha spec") # Test the method - expect(spec_service.sync_specs_with_source('org/repo', 'sha123', 'url')).to be false + expect(spec_service.sync_specs_with_source('org/repo', 'sha123', 'url')).to be true end end end \ No newline at end of file From b5fe86c9d17f6de909dea643369f5b0a2944cf0e Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 16:42:59 -0500 Subject: [PATCH 14/34] Set required ruby version in gemspec --- Gemfile | 4 ---- grade_runner.gemspec | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 9176fd9..bbd051e 100644 --- a/Gemfile +++ b/Gemfile @@ -1,9 +1,5 @@ source "https://rubygems.org" -# TODO: will this break projects on different ruby versions? can we set a range? -# Note: grade_runner does not support 3.4 (BigDecimal) -ruby "3.2.1" - gem "activesupport", ">= 2.3.5" gem "oj", "~> 3.13.12" gem "octokit", "~> 5.0" diff --git a/grade_runner.gemspec b/grade_runner.gemspec index 29e1675..88732fb 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -8,6 +8,8 @@ Gem::Specification.new do |s| s.name = "grade_runner".freeze s.version = "0.0.13" + # Note: rspec tests do not yet support 3.4 (BigDecimal) + s.required_ruby_version = Gem::Requirement.new(">= 2", "< 3.4") s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= s.require_paths = ["lib".freeze] s.authors = ["Raghu Betina".freeze, "Jelani Woods".freeze] From e10c7ebd986d123d2ddfb231a483da219bc5af3d Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 16:51:09 -0500 Subject: [PATCH 15/34] Update juwelier and gemspec for 0.0.14 --- Gemfile | 2 +- Gemfile.lock | 20 ++++++----- Rakefile | 3 ++ grade_runner.gemspec | 82 +++++++++++++++++++------------------------- 4 files changed, 51 insertions(+), 56 deletions(-) diff --git a/Gemfile b/Gemfile index bbd051e..f5c3ce3 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ group :development do gem "rspec", "~> 3.5.0" gem "rdoc", "~> 6.1" gem "bundler", "~> 2.6.7" - gem "juwelier", "~> 2.1.0" + gem "juwelier", "~> 2.4.9" gem "simplecov", ">= 0" gem "pry", "~> 0" gem "pry-byebug", "~> 3" diff --git a/Gemfile.lock b/Gemfile.lock index 61f9f3f..0f32768 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,17 +56,21 @@ GEM i18n (1.14.1) concurrent-ruby (~> 1.0) interception (0.5) - juwelier (2.1.3) + juwelier (2.4.9) builder - bundler (>= 1.13) - git (>= 1.2.5) + bundler + git github_api - highline (>= 1.6.15) - nokogiri (>= 1.5.10) + highline + kamelcase (~> 0) + nokogiri + psych rake rdoc - semver + semver2 jwt (2.7.0) + kamelcase (0.0.2) + semver2 (~> 3) method_source (1.0.0) mini_portile2 (2.8.1) minitest (5.19.0) @@ -130,7 +134,7 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) - semver (1.0.1) + semver2 (3.4.2) simplecov (0.21.2) docile (~> 1.1) simplecov-html (~> 0.11) @@ -154,7 +158,7 @@ DEPENDENCIES activesupport (>= 2.3.5) bundler (~> 2.6.7) faraday-retry (~> 1.0.3) - juwelier (~> 2.1.0) + juwelier (~> 2.4.9) octokit (~> 5.0) oj (~> 3.13.12) pry (~> 0) diff --git a/Rakefile b/Rakefile index 1d612ce..fcca038 100644 --- a/Rakefile +++ b/Rakefile @@ -21,6 +21,9 @@ Juwelier::Tasks.new do |gem| gem.email = ["raghu@firstdraft.com", "jelani@firstdraft.com"] gem.authors = ["Raghu Betina", "Jelani Woods"] + # Note: rspec tests do not yet support 3.4 (BigDecimal) + gem.required_ruby_version = Gem::Requirement.new(">= 2", "< 3.4") + # dependencies defined in Gemfile end Juwelier::RubygemsDotOrgTasks.new diff --git a/grade_runner.gemspec b/grade_runner.gemspec index 88732fb..a835d24 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -2,18 +2,16 @@ # DO NOT EDIT THIS FILE DIRECTLY # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec' # -*- encoding: utf-8 -*- -# stub: grade_runner 0.0.13 ruby lib +# stub: grade_runner 0.0.14 ruby lib Gem::Specification.new do |s| s.name = "grade_runner".freeze - s.version = "0.0.13" + s.version = "0.0.14" - # Note: rspec tests do not yet support 3.4 (BigDecimal) - s.required_ruby_version = Gem::Requirement.new(">= 2", "< 3.4") s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= s.require_paths = ["lib".freeze] s.authors = ["Raghu Betina".freeze, "Jelani Woods".freeze] - s.date = "2025-03-17" + s.date = "2025-04-03" s.description = "This gem runs your RSpec test suite and posts the JSON output to grades.firstdraft.com.".freeze s.email = ["raghu@firstdraft.com".freeze, "jelani@firstdraft.com".freeze] s.extra_rdoc_files = [ @@ -31,56 +29,46 @@ Gem::Specification.new do |s| "VERSION", "grade_runner.gemspec", "lib/grade_runner.rb", + "lib/grade_runner/formatters/hint_formatter.rb", + "lib/grade_runner/formatters/json_output_formatter.rb", "lib/grade_runner/railtie.rb", "lib/grade_runner/runner.rb", + "lib/grade_runner/services/config_service.rb", + "lib/grade_runner/services/github_service.rb", + "lib/grade_runner/services/grade_service.rb", + "lib/grade_runner/services/spec_service.rb", + "lib/grade_runner/services/token_service.rb", + "lib/grade_runner/utils/path_utils.rb", "lib/tasks/grade.rake", "lib/tasks/grade_runner.rake", - "spec/spec_helper.rb" + "spec/grade_service_spec.rb", + "spec/spec_helper.rb", + "spec/spec_service_spec.rb" ] s.homepage = "http://github.com/firstdraft/grade_runner".freeze s.licenses = ["MIT".freeze] - s.rubygems_version = "3.1.6".freeze + s.required_ruby_version = Gem::Requirement.new([">= 2".freeze, "< 3.4".freeze]) + s.rubygems_version = "3.4.6".freeze s.summary = "A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)".freeze - if s.respond_to? :specification_version then - s.specification_version = 4 - end + s.specification_version = 4 - if s.respond_to? :add_runtime_dependency then - s.add_runtime_dependency(%q.freeze, [">= 2.3.5"]) - s.add_runtime_dependency(%q.freeze, ["~> 3.13.12"]) - s.add_runtime_dependency(%q.freeze, ["~> 5.0"]) - s.add_runtime_dependency(%q.freeze, [">= 0"]) - s.add_runtime_dependency(%q.freeze, ["~> 1.0.3"]) - s.add_runtime_dependency(%q.freeze, ["~> 13"]) - s.add_development_dependency(%q.freeze, ["~> 3.5.0"]) - s.add_development_dependency(%q.freeze, ["~> 6.1"]) - s.add_development_dependency(%q.freeze, ["~> 2.1.4"]) - s.add_development_dependency(%q.freeze, ["~> 2.1.0"]) - s.add_development_dependency(%q.freeze, [">= 0"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - s.add_development_dependency(%q.freeze, ["~> 3"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - s.add_development_dependency(%q.freeze, ["~> 1"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - else - s.add_dependency(%q.freeze, [">= 2.3.5"]) - s.add_dependency(%q.freeze, ["~> 3.13.12"]) - s.add_dependency(%q.freeze, ["~> 5.0"]) - s.add_dependency(%q.freeze, [">= 0"]) - s.add_dependency(%q.freeze, ["~> 1.0.3"]) - s.add_dependency(%q.freeze, ["~> 13"]) - s.add_dependency(%q.freeze, ["~> 3.5.0"]) - s.add_dependency(%q.freeze, ["~> 6.1"]) - s.add_dependency(%q.freeze, ["~> 2.1.4"]) - s.add_dependency(%q.freeze, ["~> 2.1.0"]) - s.add_dependency(%q.freeze, [">= 0"]) - s.add_dependency(%q.freeze, ["~> 0"]) - s.add_dependency(%q.freeze, ["~> 3"]) - s.add_dependency(%q.freeze, ["~> 0"]) - s.add_dependency(%q.freeze, ["~> 0"]) - s.add_dependency(%q.freeze, ["~> 1"]) - s.add_dependency(%q.freeze, ["~> 0"]) - end + s.add_runtime_dependency(%q.freeze, [">= 2.3.5"]) + s.add_runtime_dependency(%q.freeze, ["~> 3.13.12"]) + s.add_runtime_dependency(%q.freeze, ["~> 5.0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, ["~> 1.0.3"]) + s.add_runtime_dependency(%q.freeze, ["~> 13"]) + s.add_development_dependency(%q.freeze, ["~> 3.5.0"]) + s.add_development_dependency(%q.freeze, ["~> 6.1"]) + s.add_development_dependency(%q.freeze, ["~> 2.6.7"]) + s.add_development_dependency(%q.freeze, ["~> 2.4.9"]) + s.add_development_dependency(%q.freeze, [">= 0"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) + s.add_development_dependency(%q.freeze, ["~> 3"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) + s.add_development_dependency(%q.freeze, ["~> 1"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) end + From 0a07f2776af2a08eaf5b079e6f0b26d36ef4cae4 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 3 Apr 2025 16:55:39 -0500 Subject: [PATCH 16/34] Fix newlines and formatting --- lib/grade_runner/runner.rb | 2 +- lib/grade_runner/services/config_service.rb | 2 +- lib/grade_runner/services/github_service.rb | 8 ++-- lib/grade_runner/services/spec_service.rb | 44 ++++++++++----------- lib/grade_runner/utils/path_utils.rb | 2 +- lib/tasks/grade_runner.rake | 4 +- spec/grade_service_spec.rb | 28 ++++++------- spec/spec_service_spec.rb | 10 ++--- 8 files changed, 50 insertions(+), 50 deletions(-) diff --git a/lib/grade_runner/runner.rb b/lib/grade_runner/runner.rb index bcc5d5c..13f300e 100644 --- a/lib/grade_runner/runner.rb +++ b/lib/grade_runner/runner.rb @@ -47,4 +47,4 @@ def data } end end -end \ No newline at end of file +end diff --git a/lib/grade_runner/services/config_service.rb b/lib/grade_runner/services/config_service.rb index bab3224..668a891 100644 --- a/lib/grade_runner/services/config_service.rb +++ b/lib/grade_runner/services/config_service.rb @@ -52,4 +52,4 @@ def clear_token_in_config(config_file_name) end end end -end \ No newline at end of file +end diff --git a/lib/grade_runner/services/github_service.rb b/lib/grade_runner/services/github_service.rb index e2e66d9..1c27c60 100644 --- a/lib/grade_runner/services/github_service.rb +++ b/lib/grade_runner/services/github_service.rb @@ -16,14 +16,14 @@ def retrieve_github_username(config_file_name) github_email = `git config user.email`.chomp return "" if github_email.blank? - + username = `git config user.name`.chomp search_results = Octokit.search_users("#{github_email} in:email").fetch(:items) - + if search_results.present? username = search_results.first.fetch(:login, username) end - + username end @@ -46,4 +46,4 @@ def get_repo_name end end end -end \ No newline at end of file +end diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb index 63f1b04..a8c96c0 100644 --- a/lib/grade_runner/services/spec_service.rb +++ b/lib/grade_runner/services/spec_service.rb @@ -15,10 +15,10 @@ def initialize def run_tests(output_path) # Ensure database is migrated if Rails app `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - + # Run tests with JSON formatter `RAILS_ENV=test bundle exec rspec --format JsonOutputFormatter --out #{output_path}` - + # Load and return test results Oj.load(File.read(output_path)) end @@ -26,23 +26,23 @@ def run_tests(output_path) def sync_specs_with_source(full_reponame, remote_sha, repo_url) # Return if required parameters are missing return false unless full_reponame && remote_sha && repo_url - + # Unstage staged changes in spec folder `git restore --staged spec/* 2>/dev/null` # Discard unstaged changes in spec folder `git checkout spec -q 2>/dev/null` `git clean spec -f -q 2>/dev/null` - + # Get local SHA of spec folder local_sha_output = `git ls-tree HEAD #{@path_utils.project_root}/spec 2>/dev/null`.chomp local_sha = local_sha_output.split[2] if local_sha_output.present? - + # Only update if remote SHA differs from local SHA unless remote_sha == local_sha # Create temporary directories tmp_dir = @path_utils.find_or_create_directory("tmp") backup_dir = @path_utils.find_or_create_directory("tmp/backup") - + # Backup existing specs if they exist if Dir.exist?("#{@path_utils.project_root}/spec") files_and_subfolders = Dir.glob("#{@path_utils.project_root}/spec/*") @@ -50,30 +50,30 @@ def sync_specs_with_source(full_reponame, remote_sha, repo_url) else FileUtils.mkdir_p("#{@path_utils.project_root}/spec") end - + # Download spec zip file zip_path = "#{tmp_dir}/spec.zip" download_file(repo_url, zip_path) - + # Extract zip file extracted_folder = extract_zip(zip_path, tmp_dir) source_directory = File.join(extracted_folder, "spec") - + # Overwrite spec folder with new specs overwrite_spec_folder(source_directory) - + # Clean up FileUtils.rm(zip_path) if File.exist?(zip_path) FileUtils.rm_rf(extracted_folder) if extracted_folder && Dir.exist?(extracted_folder) FileUtils.rm_rf(backup_dir) if Dir.exist?(backup_dir) - + # Commit changes `git add spec/ 2>/dev/null` `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft " 2>/dev/null` - + return true end - + false rescue => e # In case of error, still try to clean up @@ -82,37 +82,37 @@ def sync_specs_with_source(full_reponame, remote_sha, repo_url) FileUtils.rm_rf("#{@path_utils.tmp_path}/backup") if Dir.exist?("#{@path_utils.tmp_path}/backup") false end - + def download_file(url, destination) download = URI.open(url) IO.copy_stream(download, destination) end - + def extract_zip(folder, destination) extracted_file_path = destination - + Zip::File.open(folder) do |zip_file| zip_file.each_with_index do |file, index| # Get name of root folder in zip file if index == 0 extracted_file_path = File.join(destination, file.name) end - + file_path = File.join(destination, file.name) FileUtils.mkdir_p(File.dirname(file_path)) file.extract(file_path) unless File.exist?(file_path) end end - + extracted_file_path end - + def overwrite_spec_folder(source_directory) destination_directory = "#{@path_utils.project_root}/spec" - + # Get all files in the source directory files = Dir.glob("#{source_directory}/*") - + # Move each file to the destination directory files.each do |file| FileUtils.cp_r(file, destination_directory) @@ -124,4 +124,4 @@ def prepare_output_directory end end end -end \ No newline at end of file +end diff --git a/lib/grade_runner/utils/path_utils.rb b/lib/grade_runner/utils/path_utils.rb index a229985..45ef49c 100644 --- a/lib/grade_runner/utils/path_utils.rb +++ b/lib/grade_runner/utils/path_utils.rb @@ -45,4 +45,4 @@ def tmp_path end end end -end \ No newline at end of file +end diff --git a/lib/tasks/grade_runner.rake b/lib/tasks/grade_runner.rake index a7e84eb..37b87d1 100644 --- a/lib/tasks/grade_runner.rake +++ b/lib/tasks/grade_runner.rake @@ -10,7 +10,7 @@ namespace :grade_runner do reponame = ENV["CIRCLE_PROJECT_REPONAME"] sha = ENV["CIRCLE_SHA1"] token = ENV['GRADES_PERSONAL_ACCESS_TOKEN'] - + if token.present? # If in CI environment, use Runner directly with CI parameters submission_url = GradeRunner.submission_url @@ -19,4 +19,4 @@ namespace :grade_runner do puts "We couldn't find your access token, so we couldn't record your grade. Please click on the assignment link again and run the rails grade ... command shown there." end end -end \ No newline at end of file +end diff --git a/spec/grade_service_spec.rb b/spec/grade_service_spec.rb index ca99c00..40702f5 100644 --- a/spec/grade_service_spec.rb +++ b/spec/grade_service_spec.rb @@ -2,11 +2,11 @@ describe GradeRunner::Services::GradeService do let(:grade_service) { GradeRunner::Services::GradeService.new } - + before do # Save original value @original_override_value = GradeRunner.override_local_specs - + # Mock dependencies allow_any_instance_of(GradeRunner::Services::ConfigService).to receive(:get_config_file_path).and_return('config_file.yml') allow_any_instance_of(GradeRunner::Services::ConfigService).to receive(:load_config).and_return({ 'personal_access_token' => 'token' }) @@ -14,57 +14,57 @@ allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:validate_token).and_return(true) allow_any_instance_of(GradeRunner::Services::GithubService).to receive(:retrieve_github_username).and_return('username') end - + after do # Restore original value GradeRunner.override_local_specs = @original_override_value end - + describe '#process_grade_all' do context 'when specs synchronization is enabled' do it 'calls sync_specs_with_source when override_local_specs is true' do # Set up configuration GradeRunner.override_local_specs = true - + # Mock token fetch to avoid API calls resource_info = { 'repo_slug' => 'org/repo', 'spec_folder_sha' => 'sha123', 'source_code_url' => 'url' } - + allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:fetch_upstream_repo).and_return(resource_info) - + # Expectations expect_any_instance_of(GradeRunner::Services::GithubService).to receive(:set_upstream_remote).with('org/repo') expect_any_instance_of(GradeRunner::Services::SpecService).to receive(:sync_specs_with_source).with('org/repo', 'sha123', 'url') - + # Skip the actual test running and submission allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:prepare_output_directory).and_return('output.json') allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:run_tests).and_return({}) allow_any_instance_of(GradeRunner::Runner).to receive(:process).and_return(true) - + grade_service.process_grade_all end end - + context 'when specs synchronization is disabled' do it 'does not call sync_specs_with_source when override_local_specs is false' do # Set up configuration GradeRunner.override_local_specs = false - + # Expectations expect_any_instance_of(GradeRunner::Services::TokenService).not_to receive(:fetch_upstream_repo) expect_any_instance_of(GradeRunner::Services::GithubService).not_to receive(:set_upstream_remote) expect_any_instance_of(GradeRunner::Services::SpecService).not_to receive(:sync_specs_with_source) - + # Skip the actual test running and submission allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:prepare_output_directory).and_return('output.json') allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:run_tests).and_return({}) allow_any_instance_of(GradeRunner::Runner).to receive(:process).and_return(true) - + grade_service.process_grade_all end end end -end \ No newline at end of file +end diff --git a/spec/spec_service_spec.rb b/spec/spec_service_spec.rb index fa5097d..8de1ec7 100644 --- a/spec/spec_service_spec.rb +++ b/spec/spec_service_spec.rb @@ -22,7 +22,7 @@ allow(File).to receive(:exist?).and_return(false) allow(File).to receive(:join) { |*args| args.join('/') } end - + it 'returns false when required parameters are missing' do expect(spec_service.sync_specs_with_source(nil, 'sha123', 'url')).to be false expect(spec_service.sync_specs_with_source('org/repo', nil, 'url')).to be false @@ -33,17 +33,17 @@ it 'handles git operations properly' do # Mock all the git shell commands allow(spec_service).to receive(:`).with(any_args).and_return('') - + # Mock URI.open and zip extraction allow(spec_service).to receive(:download_file).and_return(true) allow(spec_service).to receive(:extract_zip).and_return('/tmp/mock_path/extracted') allow(spec_service).to receive(:overwrite_spec_folder).and_return(true) - + # Force the method to update specs by making SHA comparison fail allow(spec_service).to receive(:`).with(/git ls-tree/).and_return("blob 100644 different-sha spec") - + # Test the method expect(spec_service.sync_specs_with_source('org/repo', 'sha123', 'url')).to be true end end -end \ No newline at end of file +end From 228b0a3a8852413a74d30d9e94ab7feff3a1aa47 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Fri, 4 Apr 2025 12:54:11 -0500 Subject: [PATCH 17/34] Add more test coverage --- spec/config_service_spec.rb | 152 ++++++++++++++++++ spec/formatters/hint_formatter_spec.rb | 48 ++++++ spec/formatters/json_output_formatter_spec.rb | 90 +++++++++++ spec/github_service_spec.rb | 116 +++++++++++++ spec/path_utils_spec.rb | 139 ++++++++++++++++ spec/runner_spec.rb | 82 ++++++++++ spec/spec_helper.rb | 14 ++ spec/token_service_spec.rb | 152 ++++++++++++++++++ 8 files changed, 793 insertions(+) create mode 100644 spec/config_service_spec.rb create mode 100644 spec/formatters/hint_formatter_spec.rb create mode 100644 spec/formatters/json_output_formatter_spec.rb create mode 100644 spec/github_service_spec.rb create mode 100644 spec/path_utils_spec.rb create mode 100644 spec/runner_spec.rb create mode 100644 spec/token_service_spec.rb diff --git a/spec/config_service_spec.rb b/spec/config_service_spec.rb new file mode 100644 index 0000000..e605f77 --- /dev/null +++ b/spec/config_service_spec.rb @@ -0,0 +1,152 @@ +require 'spec_helper' +require 'fileutils' +require 'tempfile' + +describe GradeRunner::Services::ConfigService do + let(:config_service) { GradeRunner::Services::ConfigService.new } + let(:config_dir) { '.vscode' } + let(:config_file) { '.ltici_apitoken.yml' } + let(:config_path) { "#{config_dir}/#{config_file}" } + let(:default_url) { 'https://example.com' } + + describe '#find_or_create_directory' do + it 'delegates to PathUtils' do + expect(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).with(config_dir).and_return('/path/to/dir') + expect(config_service.find_or_create_directory(config_dir)).to eq('/path/to/dir') + end + end + + describe '#update_config_file' do + let(:temp_file) { Tempfile.new(['config', '.yml']) } + let(:config) { { 'key' => 'value' } } + + after do + temp_file.close + temp_file.unlink + end + + it 'writes YAML to the specified file' do + expect(File).to receive(:write).with(temp_file.path, YAML.dump(config)) + config_service.update_config_file(temp_file.path, config) + end + end + + describe '#load_config' do + context 'when config file exists' do + before do + allow(File).to receive(:exist?).with(config_path).and_return(true) + end + + it 'loads and returns config data' do + config_data = { 'submission_url' => 'https://example.com', 'personal_access_token' => 'token123' } + expect(YAML).to receive(:load_file).with(config_path).and_return(config_data) + + result = config_service.load_config(config_path, default_url) + expect(result).to eq(config_data) + end + + it 'handles YAML loading failures' do + # Add #red method to the String class temporarily + String.class_eval do + def red + self + end + end + + # Let YAML.load_file raise an error + expect(YAML).to receive(:load_file).with(config_path).and_raise(StandardError) + + # Redefine abort to return a value instead of exiting + original_abort = Kernel.method(:abort) + begin + Kernel.define_singleton_method(:abort) do |message| + # Just return a symbol instead of aborting + :aborted + end + + # The method should return nil or not raise an error + result = config_service.load_config(config_path, default_url) + expect(result).to eq(:aborted) + ensure + # Restore original abort method + Kernel.define_singleton_method(:abort, original_abort) + + # Remove our temporary method from String + String.class_eval do + undef :red + end + end + end + end + + context 'when config file does not exist' do + before do + allow(File).to receive(:exist?).with(config_path).and_return(false) + end + + it 'returns a hash with default submission URL' do + result = config_service.load_config(config_path, default_url) + expect(result).to eq({ 'submission_url' => default_url }) + end + end + end + + describe '#get_config_file_path' do + it 'returns the expected config file path' do + expect(config_service).to receive(:find_or_create_directory).with(config_dir).and_return("#{config_dir}") + expect(config_service.get_config_file_path).to eq(config_path) + end + + it 'accepts custom directory and filename' do + custom_dir = 'custom_dir' + custom_file = 'custom_file.yml' + expect(config_service).to receive(:find_or_create_directory).with(custom_dir).and_return(custom_dir) + expect(config_service.get_config_file_path(custom_dir, custom_file)).to eq("#{custom_dir}/#{custom_file}") + end + end + + describe '#save_token_to_config' do + let(:token) { 'test_token' } + let(:submission_url) { 'https://example.com' } + let(:github_username) { 'testuser' } + + it 'creates a config hash and updates the config file' do + expected_config = { + 'submission_url' => submission_url, + 'personal_access_token' => token, + 'github_username' => github_username + } + + expect(config_service).to receive(:update_config_file).with(config_path, expected_config) + config_service.save_token_to_config(config_path, token, submission_url, github_username) + end + end + + describe '#clear_token_in_config' do + context 'when config file exists' do + let(:existing_config) { { 'submission_url' => 'https://example.com', 'personal_access_token' => 'old_token' } } + + before do + allow(File).to receive(:exist?).with(config_path).and_return(true) + allow(YAML).to receive(:load_file).with(config_path).and_return(existing_config) + end + + it 'sets the personal_access_token to nil and updates the file' do + expected_config = existing_config.merge('personal_access_token' => nil) + expect(config_service).to receive(:update_config_file).with(config_path, expected_config) + config_service.clear_token_in_config(config_path) + end + end + + context 'when config file does not exist' do + before do + allow(File).to receive(:exist?).with(config_path).and_return(false) + end + + it 'does nothing' do + expect(config_service).not_to receive(:update_config_file) + config_service.clear_token_in_config(config_path) + end + end + end +end diff --git a/spec/formatters/hint_formatter_spec.rb b/spec/formatters/hint_formatter_spec.rb new file mode 100644 index 0000000..55a5750 --- /dev/null +++ b/spec/formatters/hint_formatter_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' +require 'stringio' + +describe HintFormatter do + let(:output) { StringIO.new } + let(:formatter) { HintFormatter.new(output) } + let(:example) { double('example') } + let(:exception) { double('exception') } + let(:notification) { RSpec::Core::Notifications::FailedExampleNotification.new(example) } + + before do + allow(example).to receive(:metadata).and_return({}) + allow(example).to receive(:execution_result).and_return(double('result', exception: exception, status: :failed)) + allow(example).to receive(:description).and_return('test example') + allow(example).to receive(:location_rerun_argument).and_return('location') + allow(example).to receive(:full_description).and_return('full test example') + allow(exception).to receive(:message).and_return('error message') + allow(exception).to receive(:class).and_return(StandardError) + allow(exception).to receive(:backtrace).and_return(['line 1', 'line 2']) + end + + describe '#example_failed' do + context 'when hint is present in metadata' do + before do + # We need to use an array for the hint since that's how it's checked in the actual code + allow(example).to receive(:metadata).and_return({ hint: ['This is a hint'] }) + allow_any_instance_of(Array).to receive(:present?).and_return(true) + end + + it 'displays the hint' do + formatter.example_failed(notification) + expect(output.string).to include('HINT:') + expect(output.string).to include('This is a hint') + end + end + + context 'when hint is not present in metadata' do + before do + allow_any_instance_of(NilClass).to receive(:present?).and_return(false) + end + + it 'does not display any hint' do + formatter.example_failed(notification) + expect(output.string).not_to include('HINT:') + end + end + end +end diff --git a/spec/formatters/json_output_formatter_spec.rb b/spec/formatters/json_output_formatter_spec.rb new file mode 100644 index 0000000..0bc1f1c --- /dev/null +++ b/spec/formatters/json_output_formatter_spec.rb @@ -0,0 +1,90 @@ +require 'spec_helper' +require 'stringio' +require 'tempfile' +require 'json' + +describe JsonOutputFormatter do + let(:output) { StringIO.new } + let(:formatter) { JsonOutputFormatter.new(output) } + let(:example_group) { RSpec::Core::ExampleGroup.describe('TestGroup') } + + before do + # Set up GradeRunner.default_points + allow(GradeRunner).to receive(:default_points).and_return(1) + end + + describe '#dump_summary' do + let(:summary) do + instance_double( + RSpec::Core::Notifications::SummaryNotification, + duration: 1.5, + example_count: 3, + failure_count: 1, + pending_count: 1, + errors_outside_of_examples_count: 0, + examples: [ + double('example1', + metadata: { points: 2 }, + execution_result: double('result1', status: :passed) + ), + double('example2', + metadata: { points: 3 }, + execution_result: double('result2', status: :failed) + ), + double('example3', + metadata: {}, + execution_result: double('result3', status: :pending) + ) + ] + ) + end + + before do + formatter.instance_variable_set(:@output_hash, { examples: [] }) + end + + it 'calculates and formats the summary correctly' do + formatter.dump_summary(summary) + formatter.close(nil) + + result = Oj.load(output.string) + + expect(result['summary']).to include( + 'duration' => 1.5, + 'example_count' => 3, + 'failure_count' => 1, + 'pending_count' => 1, + 'total_points' => 6, # 2 + 3 + 1 (default) + 'earned_points' => 2 # only the passed test counts + ) + + expect(result['summary']['score']).to be_within(0.0001).of(0.3333) + expect(result['summary_line']).to include('3 tests') + expect(result['summary_line']).to include('1 failures') + expect(result['summary_line']).to include('2/6 points') + expect(result['summary_line']).to include('33.33%') + end + + context 'when there are errors outside examples' do + let(:summary_with_errors) do + instance_double( + RSpec::Core::Notifications::SummaryNotification, + duration: 1.5, + example_count: 3, + failure_count: 1, + pending_count: 1, + errors_outside_of_examples_count: 1, + examples: [] + ) + end + + it 'sets a special error message in the result' do + formatter.dump_summary(summary_with_errors) + formatter.close(nil) + + result = Oj.load(output.string) + expect(result['summary_line']).to include('An error occurred while running tests') + end + end + end +end diff --git a/spec/github_service_spec.rb b/spec/github_service_spec.rb new file mode 100644 index 0000000..e7bcc19 --- /dev/null +++ b/spec/github_service_spec.rb @@ -0,0 +1,116 @@ +require 'spec_helper' +require 'octokit' +require 'yaml' + +describe GradeRunner::Services::GithubService do + let(:github_service) { GradeRunner::Services::GithubService.new } + let(:config_file_path) { '/path/to/config.yml' } + + describe '#retrieve_github_username' do + context 'when username is in config file' do + before do + allow(File).to receive(:exist?).with(config_file_path).and_return(true) + allow(YAML).to receive(:load_file).with(config_file_path).and_return({ + 'github_username' => 'config_username' + }) + end + + it 'returns username from config file' do + expect(github_service.retrieve_github_username(config_file_path)).to eq('config_username') + end + end + + context 'when username is not in config file' do + before do + allow(File).to receive(:exist?).with(config_file_path).and_return(true) + allow(YAML).to receive(:load_file).with(config_file_path).and_return({}) + allow(github_service).to receive(:`).with('git config user.email').and_return("user@example.com\n") + allow(github_service).to receive(:`).with('git config user.name').and_return("Local Username\n") + end + + it 'searches GitHub for email and returns GitHub username if found' do + search_response = { + items: [ + { login: 'github_username' } + ] + } + allow(Octokit).to receive(:search_users).with('user@example.com in:email').and_return(search_response) + + expect(github_service.retrieve_github_username(config_file_path)).to eq('github_username') + end + + it 'returns git config username if GitHub search returns no results' do + search_response = { items: [] } + allow(Octokit).to receive(:search_users).with('user@example.com in:email').and_return(search_response) + + expect(github_service.retrieve_github_username(config_file_path)).to eq('Local Username') + end + + it 'returns empty string if git email is blank' do + allow(github_service).to receive(:`).with('git config user.email').and_return("\n") + + expect(github_service.retrieve_github_username(config_file_path)).to eq('') + end + end + + context 'when config file does not exist' do + before do + allow(File).to receive(:exist?).with(config_file_path).and_return(false) + allow(github_service).to receive(:`).with('git config user.email').and_return("user@example.com\n") + allow(github_service).to receive(:`).with('git config user.name').and_return("Local Username\n") + end + + it 'falls back to git config and GitHub search' do + search_response = { + items: [ + { login: 'github_username' } + ] + } + allow(Octokit).to receive(:search_users).with('user@example.com in:email').and_return(search_response) + + expect(github_service.retrieve_github_username(config_file_path)).to eq('github_username') + end + end + end + + describe '#set_upstream_remote' do + let(:repo_slug) { 'organization/repo-name' } + let(:upstream_url) { "https://github.com/#{repo_slug}" } + + context 'when upstream remote does not exist' do + before do + allow(github_service).to receive(:`).with('git remote -v | grep -w upstream').and_return('') + end + + it 'adds a new upstream remote' do + expect(github_service).to receive(:`).with("git remote add upstream #{upstream_url}") + github_service.set_upstream_remote(repo_slug) + end + end + + context 'when upstream remote already exists' do + before do + allow(github_service).to receive(:`).with('git remote -v | grep -w upstream').and_return('upstream https://github.com/old/repo (fetch)') + end + + it 'updates the existing upstream remote' do + expect(github_service).to receive(:`).with("git remote set-url upstream #{upstream_url}") + github_service.set_upstream_remote(repo_slug) + end + end + end + + describe '#get_commit_sha' do + it 'returns the first 8 characters of the current commit SHA' do + allow(github_service).to receive(:`).with('git rev-parse HEAD').and_return('1234567890abcdef1234567890abcdef12345678') + expect(github_service.get_commit_sha).to eq('12345678') + end + end + + describe '#get_repo_name' do + it 'returns the repository name from the project path' do + allow(GradeRunner::Utils::PathUtils).to receive(:project_root).and_return(Pathname.new('/path/to/repository-name')) + expect(github_service.get_repo_name).to eq('repository-name') + end + end +end diff --git a/spec/path_utils_spec.rb b/spec/path_utils_spec.rb new file mode 100644 index 0000000..b93660c --- /dev/null +++ b/spec/path_utils_spec.rb @@ -0,0 +1,139 @@ +require 'spec_helper' +require 'fileutils' +require 'pathname' + +describe GradeRunner::Utils::PathUtils do + describe '.project_root' do + context 'when Rails is defined' do + before do + # Create a mock Rails module + module Rails + def self.root + Pathname.new('/rails/root') + end + end + # Save original constant + @original_rails = Rails if defined?(Rails) + end + + after do + # Clean up our mock + Object.send(:remove_const, :Rails) + # Restore original if it existed + Rails = @original_rails if @original_rails + end + + it 'returns Rails.root path' do + expect(GradeRunner::Utils::PathUtils.project_root).to eq(Pathname.new('/rails/root')) + end + end + + context 'when Bundler is defined but Rails is not' do + before do + # Remove Rails for this test + if defined?(Rails) + @original_rails = Rails + Object.send(:remove_const, :Rails) + end + + # Create a mock Bundler module + module Bundler + def self.root + Pathname.new('/bundler/root') + end + end + # Save original constant + @original_bundler = Bundler if defined?(Bundler) + end + + after do + # Clean up our mock + Object.send(:remove_const, :Bundler) + # Restore original if it existed + Bundler = @original_bundler if @original_bundler + # Restore Rails if it was removed + Rails = @original_rails if @original_rails + end + + it 'returns Bundler.root path' do + expect(GradeRunner::Utils::PathUtils.project_root).to eq(Pathname.new('/bundler/root')) + end + end + + context 'when neither Rails nor Bundler is defined' do + before do + # Remove constants for this test + if defined?(Rails) + @original_rails = Rails + Object.send(:remove_const, :Rails) + end + + if defined?(Bundler) + @original_bundler = Bundler + Object.send(:remove_const, :Bundler) + end + + allow(Dir).to receive(:pwd).and_return('/current/directory') + end + + after do + # Restore constants if they existed + Rails = @original_rails if @original_rails + Bundler = @original_bundler if @original_bundler + end + + it 'returns current directory as Pathname' do + expect(GradeRunner::Utils::PathUtils.project_root).to eq(Pathname.new('/current/directory')) + end + end + end + + describe '.path_in_project' do + before do + allow(GradeRunner::Utils::PathUtils).to receive(:project_root).and_return(Pathname.new('/project/root')) + end + + it 'joins path with project root' do + expect(GradeRunner::Utils::PathUtils.path_in_project('subdir')).to eq(Pathname.new('/project/root/subdir')) + end + end + + describe '.find_or_create_directory' do + before do + allow(GradeRunner::Utils::PathUtils).to receive(:path_in_project).and_return(Pathname.new('/project/root/test_dir')) + allow(Dir).to receive(:exist?).and_return(false) + allow(FileUtils).to receive(:mkdir_p) + end + + it 'creates directory if it does not exist' do + expect(FileUtils).to receive(:mkdir_p).with(Pathname.new('/project/root/test_dir')) + result = GradeRunner::Utils::PathUtils.find_or_create_directory('test_dir') + expect(result).to eq('/project/root/test_dir') + end + + it 'does not create directory if it exists' do + allow(Dir).to receive(:exist?).and_return(true) + expect(FileUtils).not_to receive(:mkdir_p) + result = GradeRunner::Utils::PathUtils.find_or_create_directory('test_dir') + expect(result).to eq('/project/root/test_dir') + end + end + + describe '.tmp_output_path' do + before do + allow(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).and_return('/project/root/tmp/output') + allow(Time).to receive(:now).and_return(Time.at(1234567890)) + end + + it 'returns a timestamped JSON path in the output directory' do + expect(GradeRunner::Utils::PathUtils.tmp_output_path).to eq('/project/root/tmp/output/1234567890.json') + end + end + + describe '.tmp_path' do + it 'returns the tmp directory path' do + expect(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).with('tmp').and_return('/project/root/tmp') + expect(GradeRunner::Utils::PathUtils.tmp_path).to eq('/project/root/tmp') + end + end +end diff --git a/spec/runner_spec.rb b/spec/runner_spec.rb new file mode 100644 index 0000000..dc36f64 --- /dev/null +++ b/spec/runner_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +describe GradeRunner::Runner do + let(:submission_url) { 'https://example.com' } + let(:token) { '2a3b4c5d6e7f8g9h0j2k3m4n5p' } + let(:rspec_output) do + { + 'summary' => { + 'duration' => 1.5, + 'example_count' => 10, + 'failure_count' => 2, + 'pending_count' => 1 + }, + 'examples' => [ + { 'status' => 'passed', 'description' => 'test 1' }, + { 'status' => 'failed', 'description' => 'test 2' } + ] + } + end + let(:username) { 'testuser' } + let(:reponame) { 'test-repo' } + let(:sha) { 'abc1234' } + let(:context) { 'manual' } + + let(:runner) { GradeRunner::Runner.new(submission_url, token, rspec_output, username, reponame, sha, context) } + + describe '#process' do + let(:url) { "#{submission_url}/builds" } + let(:uri) { URI.parse(url) } + let(:http_response) { instance_double(Net::HTTPResponse, body: '{"url":"https://grades.example.com/results/123"}') } + let(:http) { instance_double(Net::HTTP) } + + before do + # Set up stubs for private methods + allow(runner).to receive(:submission_path).and_return('/builds') + + # Set up HTTP request stubs + allow(URI).to receive(:parse).with(url).and_return(uri) + allow(Net::HTTP::Post).to receive(:new).with(uri, 'Content-Type' => 'application/json').and_call_original + allow(Net::HTTP).to receive(:start).with(uri.hostname, uri.port, use_ssl: true).and_yield(http) + allow(http).to receive(:request).and_return(http_response) + + # Stub JSON conversion & parsing + allow_any_instance_of(Hash).to receive(:to_json).and_return('{"json":"data"}') + allow(Oj).to receive(:load).with(http_response.body).and_return({ 'url' => 'https://grades.example.com/results/123' }) + end + + it 'makes a POST request to the submission URL with the correct parameters' do + expect(Net::HTTP::Post).to receive(:new).with(uri, 'Content-Type' => 'application/json') + runner.process + end + + it 'formats the data correctly' do + # Check the data hash format + expect(runner.send(:data)).to include( + access_token: token, + test_output: rspec_output, + commit_sha: sha, + username: username, + reponame: reponame, + source: context + ) + + runner.process + end + + it 'can handle network errors' do + # This test simply verifies that when Net::HTTP.start raises an error, + # the implementation runs through the error path + allow(Net::HTTP).to receive(:start).and_raise(StandardError) + + # Since the actual implementation doesn't handle errors, just verify + # that we're testing the right thing + expect(runner).to receive(:post_to_grades).and_call_original + + # Original implementation will raise an error, which is fine for this test + expect { + runner.process + }.to raise_error(StandardError) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6489c57..d6e1c53 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,19 @@ require 'rspec' require 'grade_runner' +require 'rspec/core/formatters/documentation_formatter' + +# Require formatters directly +require_relative '../lib/grade_runner/formatters/hint_formatter.rb' +require_relative '../lib/grade_runner/formatters/json_output_formatter.rb' RSpec.configure do |config| + # Basic configuration + config.color = true + config.order = :random + config.mock_with :rspec + + # Clear out mocks between tests + config.mock_with :rspec do |mocks| + mocks.verify_partial_doubles = true + end end diff --git a/spec/token_service_spec.rb b/spec/token_service_spec.rb new file mode 100644 index 0000000..0ac4bf3 --- /dev/null +++ b/spec/token_service_spec.rb @@ -0,0 +1,152 @@ +require 'spec_helper' +require 'net/http' + +describe GradeRunner::Services::TokenService do + let(:submission_url) { 'https://example.com' } + let(:token_service) { GradeRunner::Services::TokenService.new(submission_url) } + let(:valid_token) { '2a3b4c5d6e7f8g9h0j2k3m4n5p' } + let(:invalid_token) { 'invalid-token' } + + describe '#initialize' do + it 'sets the submission_url attribute' do + expect(token_service.submission_url).to eq(submission_url) + end + end + + describe '#validate_token' do + let(:uri) { URI.parse("#{submission_url}/submissions/validate_token?token=#{valid_token}") } + let(:http_response) { instance_double(Net::HTTPResponse, body: '{"success":true}') } + let(:http) { instance_double(Net::HTTP) } + + before do + allow(URI).to receive(:parse).and_return(uri) + allow(Net::HTTP::Get).to receive(:new).and_return(double('request')) + allow(Net::HTTP).to receive(:start).and_yield(http) + allow(http).to receive(:request).and_return(http_response) + allow(Oj).to receive(:load).with(http_response.body).and_return({ 'success' => true }) + end + + context 'with valid token format' do + it 'returns true when token is valid according to API' do + # Directly stub the regex validation + allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) + expect(token_service.validate_token(valid_token)).to be true + end + + it 'returns false when API returns success: false' do + # Directly stub the regex validation + allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) + allow(Oj).to receive(:load).with(http_response.body).and_return({ 'success' => false }) + expect(token_service.validate_token(valid_token)).to be false + end + + it 'handles network errors gracefully' do + # Directly stub the regex validation + allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) + allow(Net::HTTP).to receive(:start).and_raise(StandardError) + expect(token_service.validate_token(valid_token)).to be false + end + end + + context 'with invalid token format' do + it 'returns false for nil token' do + expect(token_service.validate_token(nil)).to be false + end + + it 'returns false for non-string token' do + expect(token_service.validate_token(123)).to be false + end + + it 'returns false for invalid format token' do + expect(token_service.validate_token(invalid_token)).to be false + end + end + end + + describe '#get_token' do + let(:config_file) { 'config.yml' } + + context 'when input token is present' do + it 'returns the input token' do + expect(token_service.get_token('input_token', 'file_token', config_file)).to eq('input_token') + end + end + + context 'when input token is blank but file token is present' do + it 'returns the file token' do + expect(token_service.get_token(nil, 'file_token', config_file)).to eq('file_token') + expect(token_service.get_token('', 'file_token', config_file)).to eq('file_token') + end + end + + context 'when both input and file tokens are blank' do + it 'prompts for token' do + expect(token_service).to receive(:prompt_for_token).with(config_file).and_return('new_token') + expect(token_service.get_token(nil, nil, config_file)).to eq('new_token') + end + end + end + + describe '#prompt_for_token' do + let(:config_file) { 'config.yml' } + let(:stdin_mock) { StringIO.new(valid_token) } + + before do + $stdout = StringIO.new + $stdin = stdin_mock + end + + after do + $stdout = STDOUT + $stdin = STDIN + end + + it 'prompts for token and returns it when valid' do + allow(token_service).to receive(:validate_token).with(valid_token).and_return(true) + expect(token_service.prompt_for_token(config_file)).to eq(valid_token) + end + + it 'continues prompting until valid token is entered' do + $stdin = StringIO.new("#{invalid_token}\n#{valid_token}") + allow(token_service).to receive(:validate_token).with(invalid_token).and_return(false) + allow(token_service).to receive(:validate_token).with(valid_token).and_return(true) + expect(token_service.prompt_for_token(config_file)).to eq(valid_token) + end + end + + describe '#fetch_upstream_repo' do + let(:uri) { URI.parse("#{submission_url}/submissions/resource?token=#{valid_token}") } + let(:http_response) { instance_double(Net::HTTPResponse, body: '{"repo_slug":"org/repo","spec_folder_sha":"abc123","source_code_url":"https://example.com/archive.zip"}') } + let(:http) { instance_double(Net::HTTP) } + let(:expected_result) { { 'repo_slug' => 'org/repo', 'spec_folder_sha' => 'abc123', 'source_code_url' => 'https://example.com/archive.zip' } } + + before do + allow(URI).to receive(:parse).and_return(uri) + allow(Net::HTTP::Get).to receive(:new).and_return(double('request')) + allow(Net::HTTP).to receive(:start).and_yield(http) + allow(http).to receive(:request).and_return(http_response) + allow(Oj).to receive(:load).with(http_response.body).and_return(expected_result) + end + + context 'with valid token format' do + it 'returns repository information hash' do + # Directly stub the regex validation + allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) + expect(token_service.fetch_upstream_repo(valid_token)).to eq(expected_result) + end + + it 'handles network errors gracefully' do + # Directly stub the regex validation + allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) + allow(Net::HTTP).to receive(:start).and_raise(StandardError) + expect(token_service.fetch_upstream_repo(valid_token)).to be false + end + end + + context 'with invalid token format' do + it 'returns false' do + expect(token_service.fetch_upstream_repo(invalid_token)).to be false + end + end + end +end From b2d76b04e34fe2859579d18ca2b7affa9a90d497 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Fri, 4 Apr 2025 14:40:35 -0500 Subject: [PATCH 18/34] Remove rspec test suite for now --- .../formatters/json_output_formatter.rb | 12 +- spec/config_service_spec.rb | 152 ------------------ spec/formatters/hint_formatter_spec.rb | 48 ------ spec/formatters/json_output_formatter_spec.rb | 90 ----------- spec/github_service_spec.rb | 116 ------------- spec/grade_service_spec.rb | 70 -------- spec/path_utils_spec.rb | 139 ---------------- spec/runner_spec.rb | 82 ---------- spec/spec_helper.rb | 14 -- spec/spec_service_spec.rb | 49 ------ spec/token_service_spec.rb | 152 ------------------ 11 files changed, 6 insertions(+), 918 deletions(-) delete mode 100644 spec/config_service_spec.rb delete mode 100644 spec/formatters/hint_formatter_spec.rb delete mode 100644 spec/formatters/json_output_formatter_spec.rb delete mode 100644 spec/github_service_spec.rb delete mode 100644 spec/grade_service_spec.rb delete mode 100644 spec/path_utils_spec.rb delete mode 100644 spec/runner_spec.rb delete mode 100644 spec/spec_service_spec.rb delete mode 100644 spec/token_service_spec.rb diff --git a/lib/grade_runner/formatters/json_output_formatter.rb b/lib/grade_runner/formatters/json_output_formatter.rb index f73fea9..86b5410 100644 --- a/lib/grade_runner/formatters/json_output_formatter.rb +++ b/lib/grade_runner/formatters/json_output_formatter.rb @@ -8,7 +8,7 @@ def dump_summary(summary) examples. map { |example| example.metadata.fetch(:points, GradeRunner.default_points).to_i }. sum - + earned_points = summary. examples. select { |example| example.execution_result.status == :passed }. @@ -17,7 +17,7 @@ def dump_summary(summary) score = (earned_points.to_f / total_points).round(4) score = 0 if score.nan? - + @output_hash[:summary] = { duration: summary.duration, example_count: summary.example_count, @@ -29,14 +29,14 @@ def dump_summary(summary) score: score } result = (@output_hash[:summary][:score] * 100).round(2) - + if summary.errors_outside_of_examples_count.positive? result = "An error occurred while running tests" else result = result.to_s + "%" end - - + + @output_hash[:summary_line] = [ "#{summary.example_count} #{summary.example_count == 1 ? "test" : "tests"}", "#{summary.failure_count} failures", @@ -48,7 +48,7 @@ def dump_summary(summary) def close(_notification) output.write Oj.dump @output_hash end - + private def format_example(example) diff --git a/spec/config_service_spec.rb b/spec/config_service_spec.rb deleted file mode 100644 index e605f77..0000000 --- a/spec/config_service_spec.rb +++ /dev/null @@ -1,152 +0,0 @@ -require 'spec_helper' -require 'fileutils' -require 'tempfile' - -describe GradeRunner::Services::ConfigService do - let(:config_service) { GradeRunner::Services::ConfigService.new } - let(:config_dir) { '.vscode' } - let(:config_file) { '.ltici_apitoken.yml' } - let(:config_path) { "#{config_dir}/#{config_file}" } - let(:default_url) { 'https://example.com' } - - describe '#find_or_create_directory' do - it 'delegates to PathUtils' do - expect(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).with(config_dir).and_return('/path/to/dir') - expect(config_service.find_or_create_directory(config_dir)).to eq('/path/to/dir') - end - end - - describe '#update_config_file' do - let(:temp_file) { Tempfile.new(['config', '.yml']) } - let(:config) { { 'key' => 'value' } } - - after do - temp_file.close - temp_file.unlink - end - - it 'writes YAML to the specified file' do - expect(File).to receive(:write).with(temp_file.path, YAML.dump(config)) - config_service.update_config_file(temp_file.path, config) - end - end - - describe '#load_config' do - context 'when config file exists' do - before do - allow(File).to receive(:exist?).with(config_path).and_return(true) - end - - it 'loads and returns config data' do - config_data = { 'submission_url' => 'https://example.com', 'personal_access_token' => 'token123' } - expect(YAML).to receive(:load_file).with(config_path).and_return(config_data) - - result = config_service.load_config(config_path, default_url) - expect(result).to eq(config_data) - end - - it 'handles YAML loading failures' do - # Add #red method to the String class temporarily - String.class_eval do - def red - self - end - end - - # Let YAML.load_file raise an error - expect(YAML).to receive(:load_file).with(config_path).and_raise(StandardError) - - # Redefine abort to return a value instead of exiting - original_abort = Kernel.method(:abort) - begin - Kernel.define_singleton_method(:abort) do |message| - # Just return a symbol instead of aborting - :aborted - end - - # The method should return nil or not raise an error - result = config_service.load_config(config_path, default_url) - expect(result).to eq(:aborted) - ensure - # Restore original abort method - Kernel.define_singleton_method(:abort, original_abort) - - # Remove our temporary method from String - String.class_eval do - undef :red - end - end - end - end - - context 'when config file does not exist' do - before do - allow(File).to receive(:exist?).with(config_path).and_return(false) - end - - it 'returns a hash with default submission URL' do - result = config_service.load_config(config_path, default_url) - expect(result).to eq({ 'submission_url' => default_url }) - end - end - end - - describe '#get_config_file_path' do - it 'returns the expected config file path' do - expect(config_service).to receive(:find_or_create_directory).with(config_dir).and_return("#{config_dir}") - expect(config_service.get_config_file_path).to eq(config_path) - end - - it 'accepts custom directory and filename' do - custom_dir = 'custom_dir' - custom_file = 'custom_file.yml' - expect(config_service).to receive(:find_or_create_directory).with(custom_dir).and_return(custom_dir) - expect(config_service.get_config_file_path(custom_dir, custom_file)).to eq("#{custom_dir}/#{custom_file}") - end - end - - describe '#save_token_to_config' do - let(:token) { 'test_token' } - let(:submission_url) { 'https://example.com' } - let(:github_username) { 'testuser' } - - it 'creates a config hash and updates the config file' do - expected_config = { - 'submission_url' => submission_url, - 'personal_access_token' => token, - 'github_username' => github_username - } - - expect(config_service).to receive(:update_config_file).with(config_path, expected_config) - config_service.save_token_to_config(config_path, token, submission_url, github_username) - end - end - - describe '#clear_token_in_config' do - context 'when config file exists' do - let(:existing_config) { { 'submission_url' => 'https://example.com', 'personal_access_token' => 'old_token' } } - - before do - allow(File).to receive(:exist?).with(config_path).and_return(true) - allow(YAML).to receive(:load_file).with(config_path).and_return(existing_config) - end - - it 'sets the personal_access_token to nil and updates the file' do - expected_config = existing_config.merge('personal_access_token' => nil) - expect(config_service).to receive(:update_config_file).with(config_path, expected_config) - config_service.clear_token_in_config(config_path) - end - end - - context 'when config file does not exist' do - before do - allow(File).to receive(:exist?).with(config_path).and_return(false) - end - - it 'does nothing' do - expect(config_service).not_to receive(:update_config_file) - config_service.clear_token_in_config(config_path) - end - end - end -end diff --git a/spec/formatters/hint_formatter_spec.rb b/spec/formatters/hint_formatter_spec.rb deleted file mode 100644 index 55a5750..0000000 --- a/spec/formatters/hint_formatter_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -require 'spec_helper' -require 'stringio' - -describe HintFormatter do - let(:output) { StringIO.new } - let(:formatter) { HintFormatter.new(output) } - let(:example) { double('example') } - let(:exception) { double('exception') } - let(:notification) { RSpec::Core::Notifications::FailedExampleNotification.new(example) } - - before do - allow(example).to receive(:metadata).and_return({}) - allow(example).to receive(:execution_result).and_return(double('result', exception: exception, status: :failed)) - allow(example).to receive(:description).and_return('test example') - allow(example).to receive(:location_rerun_argument).and_return('location') - allow(example).to receive(:full_description).and_return('full test example') - allow(exception).to receive(:message).and_return('error message') - allow(exception).to receive(:class).and_return(StandardError) - allow(exception).to receive(:backtrace).and_return(['line 1', 'line 2']) - end - - describe '#example_failed' do - context 'when hint is present in metadata' do - before do - # We need to use an array for the hint since that's how it's checked in the actual code - allow(example).to receive(:metadata).and_return({ hint: ['This is a hint'] }) - allow_any_instance_of(Array).to receive(:present?).and_return(true) - end - - it 'displays the hint' do - formatter.example_failed(notification) - expect(output.string).to include('HINT:') - expect(output.string).to include('This is a hint') - end - end - - context 'when hint is not present in metadata' do - before do - allow_any_instance_of(NilClass).to receive(:present?).and_return(false) - end - - it 'does not display any hint' do - formatter.example_failed(notification) - expect(output.string).not_to include('HINT:') - end - end - end -end diff --git a/spec/formatters/json_output_formatter_spec.rb b/spec/formatters/json_output_formatter_spec.rb deleted file mode 100644 index 0bc1f1c..0000000 --- a/spec/formatters/json_output_formatter_spec.rb +++ /dev/null @@ -1,90 +0,0 @@ -require 'spec_helper' -require 'stringio' -require 'tempfile' -require 'json' - -describe JsonOutputFormatter do - let(:output) { StringIO.new } - let(:formatter) { JsonOutputFormatter.new(output) } - let(:example_group) { RSpec::Core::ExampleGroup.describe('TestGroup') } - - before do - # Set up GradeRunner.default_points - allow(GradeRunner).to receive(:default_points).and_return(1) - end - - describe '#dump_summary' do - let(:summary) do - instance_double( - RSpec::Core::Notifications::SummaryNotification, - duration: 1.5, - example_count: 3, - failure_count: 1, - pending_count: 1, - errors_outside_of_examples_count: 0, - examples: [ - double('example1', - metadata: { points: 2 }, - execution_result: double('result1', status: :passed) - ), - double('example2', - metadata: { points: 3 }, - execution_result: double('result2', status: :failed) - ), - double('example3', - metadata: {}, - execution_result: double('result3', status: :pending) - ) - ] - ) - end - - before do - formatter.instance_variable_set(:@output_hash, { examples: [] }) - end - - it 'calculates and formats the summary correctly' do - formatter.dump_summary(summary) - formatter.close(nil) - - result = Oj.load(output.string) - - expect(result['summary']).to include( - 'duration' => 1.5, - 'example_count' => 3, - 'failure_count' => 1, - 'pending_count' => 1, - 'total_points' => 6, # 2 + 3 + 1 (default) - 'earned_points' => 2 # only the passed test counts - ) - - expect(result['summary']['score']).to be_within(0.0001).of(0.3333) - expect(result['summary_line']).to include('3 tests') - expect(result['summary_line']).to include('1 failures') - expect(result['summary_line']).to include('2/6 points') - expect(result['summary_line']).to include('33.33%') - end - - context 'when there are errors outside examples' do - let(:summary_with_errors) do - instance_double( - RSpec::Core::Notifications::SummaryNotification, - duration: 1.5, - example_count: 3, - failure_count: 1, - pending_count: 1, - errors_outside_of_examples_count: 1, - examples: [] - ) - end - - it 'sets a special error message in the result' do - formatter.dump_summary(summary_with_errors) - formatter.close(nil) - - result = Oj.load(output.string) - expect(result['summary_line']).to include('An error occurred while running tests') - end - end - end -end diff --git a/spec/github_service_spec.rb b/spec/github_service_spec.rb deleted file mode 100644 index e7bcc19..0000000 --- a/spec/github_service_spec.rb +++ /dev/null @@ -1,116 +0,0 @@ -require 'spec_helper' -require 'octokit' -require 'yaml' - -describe GradeRunner::Services::GithubService do - let(:github_service) { GradeRunner::Services::GithubService.new } - let(:config_file_path) { '/path/to/config.yml' } - - describe '#retrieve_github_username' do - context 'when username is in config file' do - before do - allow(File).to receive(:exist?).with(config_file_path).and_return(true) - allow(YAML).to receive(:load_file).with(config_file_path).and_return({ - 'github_username' => 'config_username' - }) - end - - it 'returns username from config file' do - expect(github_service.retrieve_github_username(config_file_path)).to eq('config_username') - end - end - - context 'when username is not in config file' do - before do - allow(File).to receive(:exist?).with(config_file_path).and_return(true) - allow(YAML).to receive(:load_file).with(config_file_path).and_return({}) - allow(github_service).to receive(:`).with('git config user.email').and_return("user@example.com\n") - allow(github_service).to receive(:`).with('git config user.name').and_return("Local Username\n") - end - - it 'searches GitHub for email and returns GitHub username if found' do - search_response = { - items: [ - { login: 'github_username' } - ] - } - allow(Octokit).to receive(:search_users).with('user@example.com in:email').and_return(search_response) - - expect(github_service.retrieve_github_username(config_file_path)).to eq('github_username') - end - - it 'returns git config username if GitHub search returns no results' do - search_response = { items: [] } - allow(Octokit).to receive(:search_users).with('user@example.com in:email').and_return(search_response) - - expect(github_service.retrieve_github_username(config_file_path)).to eq('Local Username') - end - - it 'returns empty string if git email is blank' do - allow(github_service).to receive(:`).with('git config user.email').and_return("\n") - - expect(github_service.retrieve_github_username(config_file_path)).to eq('') - end - end - - context 'when config file does not exist' do - before do - allow(File).to receive(:exist?).with(config_file_path).and_return(false) - allow(github_service).to receive(:`).with('git config user.email').and_return("user@example.com\n") - allow(github_service).to receive(:`).with('git config user.name').and_return("Local Username\n") - end - - it 'falls back to git config and GitHub search' do - search_response = { - items: [ - { login: 'github_username' } - ] - } - allow(Octokit).to receive(:search_users).with('user@example.com in:email').and_return(search_response) - - expect(github_service.retrieve_github_username(config_file_path)).to eq('github_username') - end - end - end - - describe '#set_upstream_remote' do - let(:repo_slug) { 'organization/repo-name' } - let(:upstream_url) { "https://github.com/#{repo_slug}" } - - context 'when upstream remote does not exist' do - before do - allow(github_service).to receive(:`).with('git remote -v | grep -w upstream').and_return('') - end - - it 'adds a new upstream remote' do - expect(github_service).to receive(:`).with("git remote add upstream #{upstream_url}") - github_service.set_upstream_remote(repo_slug) - end - end - - context 'when upstream remote already exists' do - before do - allow(github_service).to receive(:`).with('git remote -v | grep -w upstream').and_return('upstream https://github.com/old/repo (fetch)') - end - - it 'updates the existing upstream remote' do - expect(github_service).to receive(:`).with("git remote set-url upstream #{upstream_url}") - github_service.set_upstream_remote(repo_slug) - end - end - end - - describe '#get_commit_sha' do - it 'returns the first 8 characters of the current commit SHA' do - allow(github_service).to receive(:`).with('git rev-parse HEAD').and_return('1234567890abcdef1234567890abcdef12345678') - expect(github_service.get_commit_sha).to eq('12345678') - end - end - - describe '#get_repo_name' do - it 'returns the repository name from the project path' do - allow(GradeRunner::Utils::PathUtils).to receive(:project_root).and_return(Pathname.new('/path/to/repository-name')) - expect(github_service.get_repo_name).to eq('repository-name') - end - end -end diff --git a/spec/grade_service_spec.rb b/spec/grade_service_spec.rb deleted file mode 100644 index 40702f5..0000000 --- a/spec/grade_service_spec.rb +++ /dev/null @@ -1,70 +0,0 @@ -require 'spec_helper' - -describe GradeRunner::Services::GradeService do - let(:grade_service) { GradeRunner::Services::GradeService.new } - - before do - # Save original value - @original_override_value = GradeRunner.override_local_specs - - # Mock dependencies - allow_any_instance_of(GradeRunner::Services::ConfigService).to receive(:get_config_file_path).and_return('config_file.yml') - allow_any_instance_of(GradeRunner::Services::ConfigService).to receive(:load_config).and_return({ 'personal_access_token' => 'token' }) - allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:get_token).and_return('valid_token') - allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:validate_token).and_return(true) - allow_any_instance_of(GradeRunner::Services::GithubService).to receive(:retrieve_github_username).and_return('username') - end - - after do - # Restore original value - GradeRunner.override_local_specs = @original_override_value - end - - describe '#process_grade_all' do - context 'when specs synchronization is enabled' do - it 'calls sync_specs_with_source when override_local_specs is true' do - # Set up configuration - GradeRunner.override_local_specs = true - - # Mock token fetch to avoid API calls - resource_info = { - 'repo_slug' => 'org/repo', - 'spec_folder_sha' => 'sha123', - 'source_code_url' => 'url' - } - - allow_any_instance_of(GradeRunner::Services::TokenService).to receive(:fetch_upstream_repo).and_return(resource_info) - - # Expectations - expect_any_instance_of(GradeRunner::Services::GithubService).to receive(:set_upstream_remote).with('org/repo') - expect_any_instance_of(GradeRunner::Services::SpecService).to receive(:sync_specs_with_source).with('org/repo', 'sha123', 'url') - - # Skip the actual test running and submission - allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:prepare_output_directory).and_return('output.json') - allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:run_tests).and_return({}) - allow_any_instance_of(GradeRunner::Runner).to receive(:process).and_return(true) - - grade_service.process_grade_all - end - end - - context 'when specs synchronization is disabled' do - it 'does not call sync_specs_with_source when override_local_specs is false' do - # Set up configuration - GradeRunner.override_local_specs = false - - # Expectations - expect_any_instance_of(GradeRunner::Services::TokenService).not_to receive(:fetch_upstream_repo) - expect_any_instance_of(GradeRunner::Services::GithubService).not_to receive(:set_upstream_remote) - expect_any_instance_of(GradeRunner::Services::SpecService).not_to receive(:sync_specs_with_source) - - # Skip the actual test running and submission - allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:prepare_output_directory).and_return('output.json') - allow_any_instance_of(GradeRunner::Services::SpecService).to receive(:run_tests).and_return({}) - allow_any_instance_of(GradeRunner::Runner).to receive(:process).and_return(true) - - grade_service.process_grade_all - end - end - end -end diff --git a/spec/path_utils_spec.rb b/spec/path_utils_spec.rb deleted file mode 100644 index b93660c..0000000 --- a/spec/path_utils_spec.rb +++ /dev/null @@ -1,139 +0,0 @@ -require 'spec_helper' -require 'fileutils' -require 'pathname' - -describe GradeRunner::Utils::PathUtils do - describe '.project_root' do - context 'when Rails is defined' do - before do - # Create a mock Rails module - module Rails - def self.root - Pathname.new('/rails/root') - end - end - # Save original constant - @original_rails = Rails if defined?(Rails) - end - - after do - # Clean up our mock - Object.send(:remove_const, :Rails) - # Restore original if it existed - Rails = @original_rails if @original_rails - end - - it 'returns Rails.root path' do - expect(GradeRunner::Utils::PathUtils.project_root).to eq(Pathname.new('/rails/root')) - end - end - - context 'when Bundler is defined but Rails is not' do - before do - # Remove Rails for this test - if defined?(Rails) - @original_rails = Rails - Object.send(:remove_const, :Rails) - end - - # Create a mock Bundler module - module Bundler - def self.root - Pathname.new('/bundler/root') - end - end - # Save original constant - @original_bundler = Bundler if defined?(Bundler) - end - - after do - # Clean up our mock - Object.send(:remove_const, :Bundler) - # Restore original if it existed - Bundler = @original_bundler if @original_bundler - # Restore Rails if it was removed - Rails = @original_rails if @original_rails - end - - it 'returns Bundler.root path' do - expect(GradeRunner::Utils::PathUtils.project_root).to eq(Pathname.new('/bundler/root')) - end - end - - context 'when neither Rails nor Bundler is defined' do - before do - # Remove constants for this test - if defined?(Rails) - @original_rails = Rails - Object.send(:remove_const, :Rails) - end - - if defined?(Bundler) - @original_bundler = Bundler - Object.send(:remove_const, :Bundler) - end - - allow(Dir).to receive(:pwd).and_return('/current/directory') - end - - after do - # Restore constants if they existed - Rails = @original_rails if @original_rails - Bundler = @original_bundler if @original_bundler - end - - it 'returns current directory as Pathname' do - expect(GradeRunner::Utils::PathUtils.project_root).to eq(Pathname.new('/current/directory')) - end - end - end - - describe '.path_in_project' do - before do - allow(GradeRunner::Utils::PathUtils).to receive(:project_root).and_return(Pathname.new('/project/root')) - end - - it 'joins path with project root' do - expect(GradeRunner::Utils::PathUtils.path_in_project('subdir')).to eq(Pathname.new('/project/root/subdir')) - end - end - - describe '.find_or_create_directory' do - before do - allow(GradeRunner::Utils::PathUtils).to receive(:path_in_project).and_return(Pathname.new('/project/root/test_dir')) - allow(Dir).to receive(:exist?).and_return(false) - allow(FileUtils).to receive(:mkdir_p) - end - - it 'creates directory if it does not exist' do - expect(FileUtils).to receive(:mkdir_p).with(Pathname.new('/project/root/test_dir')) - result = GradeRunner::Utils::PathUtils.find_or_create_directory('test_dir') - expect(result).to eq('/project/root/test_dir') - end - - it 'does not create directory if it exists' do - allow(Dir).to receive(:exist?).and_return(true) - expect(FileUtils).not_to receive(:mkdir_p) - result = GradeRunner::Utils::PathUtils.find_or_create_directory('test_dir') - expect(result).to eq('/project/root/test_dir') - end - end - - describe '.tmp_output_path' do - before do - allow(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).and_return('/project/root/tmp/output') - allow(Time).to receive(:now).and_return(Time.at(1234567890)) - end - - it 'returns a timestamped JSON path in the output directory' do - expect(GradeRunner::Utils::PathUtils.tmp_output_path).to eq('/project/root/tmp/output/1234567890.json') - end - end - - describe '.tmp_path' do - it 'returns the tmp directory path' do - expect(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).with('tmp').and_return('/project/root/tmp') - expect(GradeRunner::Utils::PathUtils.tmp_path).to eq('/project/root/tmp') - end - end -end diff --git a/spec/runner_spec.rb b/spec/runner_spec.rb deleted file mode 100644 index dc36f64..0000000 --- a/spec/runner_spec.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'spec_helper' - -describe GradeRunner::Runner do - let(:submission_url) { 'https://example.com' } - let(:token) { '2a3b4c5d6e7f8g9h0j2k3m4n5p' } - let(:rspec_output) do - { - 'summary' => { - 'duration' => 1.5, - 'example_count' => 10, - 'failure_count' => 2, - 'pending_count' => 1 - }, - 'examples' => [ - { 'status' => 'passed', 'description' => 'test 1' }, - { 'status' => 'failed', 'description' => 'test 2' } - ] - } - end - let(:username) { 'testuser' } - let(:reponame) { 'test-repo' } - let(:sha) { 'abc1234' } - let(:context) { 'manual' } - - let(:runner) { GradeRunner::Runner.new(submission_url, token, rspec_output, username, reponame, sha, context) } - - describe '#process' do - let(:url) { "#{submission_url}/builds" } - let(:uri) { URI.parse(url) } - let(:http_response) { instance_double(Net::HTTPResponse, body: '{"url":"https://grades.example.com/results/123"}') } - let(:http) { instance_double(Net::HTTP) } - - before do - # Set up stubs for private methods - allow(runner).to receive(:submission_path).and_return('/builds') - - # Set up HTTP request stubs - allow(URI).to receive(:parse).with(url).and_return(uri) - allow(Net::HTTP::Post).to receive(:new).with(uri, 'Content-Type' => 'application/json').and_call_original - allow(Net::HTTP).to receive(:start).with(uri.hostname, uri.port, use_ssl: true).and_yield(http) - allow(http).to receive(:request).and_return(http_response) - - # Stub JSON conversion & parsing - allow_any_instance_of(Hash).to receive(:to_json).and_return('{"json":"data"}') - allow(Oj).to receive(:load).with(http_response.body).and_return({ 'url' => 'https://grades.example.com/results/123' }) - end - - it 'makes a POST request to the submission URL with the correct parameters' do - expect(Net::HTTP::Post).to receive(:new).with(uri, 'Content-Type' => 'application/json') - runner.process - end - - it 'formats the data correctly' do - # Check the data hash format - expect(runner.send(:data)).to include( - access_token: token, - test_output: rspec_output, - commit_sha: sha, - username: username, - reponame: reponame, - source: context - ) - - runner.process - end - - it 'can handle network errors' do - # This test simply verifies that when Net::HTTP.start raises an error, - # the implementation runs through the error path - allow(Net::HTTP).to receive(:start).and_raise(StandardError) - - # Since the actual implementation doesn't handle errors, just verify - # that we're testing the right thing - expect(runner).to receive(:post_to_grades).and_call_original - - # Original implementation will raise an error, which is fine for this test - expect { - runner.process - }.to raise_error(StandardError) - end - end -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d6e1c53..6489c57 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,19 +1,5 @@ require 'rspec' require 'grade_runner' -require 'rspec/core/formatters/documentation_formatter' - -# Require formatters directly -require_relative '../lib/grade_runner/formatters/hint_formatter.rb' -require_relative '../lib/grade_runner/formatters/json_output_formatter.rb' RSpec.configure do |config| - # Basic configuration - config.color = true - config.order = :random - config.mock_with :rspec - - # Clear out mocks between tests - config.mock_with :rspec do |mocks| - mocks.verify_partial_doubles = true - end end diff --git a/spec/spec_service_spec.rb b/spec/spec_service_spec.rb deleted file mode 100644 index 8de1ec7..0000000 --- a/spec/spec_service_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -require 'spec_helper' -require 'fileutils' -require 'tempfile' -require 'tmpdir' - -describe GradeRunner::Services::SpecService do - let(:spec_service) { GradeRunner::Services::SpecService.new } - - describe '#sync_specs_with_source' do - before do - # Mock the path utils - allow(GradeRunner::Utils::PathUtils).to receive(:tmp_path).and_return('/tmp/mock_path') - allow(GradeRunner::Utils::PathUtils).to receive(:find_or_create_directory).and_return('/tmp/mock_path') - allow(GradeRunner::Utils::PathUtils).to receive(:project_root).and_return('/tmp/mock_project') - allow(FileUtils).to receive(:rm_rf) - allow(FileUtils).to receive(:mkdir_p) - allow(FileUtils).to receive(:mv) - allow(FileUtils).to receive(:cp_r) - allow(FileUtils).to receive(:rm) - allow(Dir).to receive(:exist?).and_return(false) - allow(Dir).to receive(:glob).and_return([]) - allow(File).to receive(:exist?).and_return(false) - allow(File).to receive(:join) { |*args| args.join('/') } - end - - it 'returns false when required parameters are missing' do - expect(spec_service.sync_specs_with_source(nil, 'sha123', 'url')).to be false - expect(spec_service.sync_specs_with_source('org/repo', nil, 'url')).to be false - expect(spec_service.sync_specs_with_source('org/repo', 'sha123', nil)).to be false - end - - # Test for git operations and spec syncing - it 'handles git operations properly' do - # Mock all the git shell commands - allow(spec_service).to receive(:`).with(any_args).and_return('') - - # Mock URI.open and zip extraction - allow(spec_service).to receive(:download_file).and_return(true) - allow(spec_service).to receive(:extract_zip).and_return('/tmp/mock_path/extracted') - allow(spec_service).to receive(:overwrite_spec_folder).and_return(true) - - # Force the method to update specs by making SHA comparison fail - allow(spec_service).to receive(:`).with(/git ls-tree/).and_return("blob 100644 different-sha spec") - - # Test the method - expect(spec_service.sync_specs_with_source('org/repo', 'sha123', 'url')).to be true - end - end -end diff --git a/spec/token_service_spec.rb b/spec/token_service_spec.rb deleted file mode 100644 index 0ac4bf3..0000000 --- a/spec/token_service_spec.rb +++ /dev/null @@ -1,152 +0,0 @@ -require 'spec_helper' -require 'net/http' - -describe GradeRunner::Services::TokenService do - let(:submission_url) { 'https://example.com' } - let(:token_service) { GradeRunner::Services::TokenService.new(submission_url) } - let(:valid_token) { '2a3b4c5d6e7f8g9h0j2k3m4n5p' } - let(:invalid_token) { 'invalid-token' } - - describe '#initialize' do - it 'sets the submission_url attribute' do - expect(token_service.submission_url).to eq(submission_url) - end - end - - describe '#validate_token' do - let(:uri) { URI.parse("#{submission_url}/submissions/validate_token?token=#{valid_token}") } - let(:http_response) { instance_double(Net::HTTPResponse, body: '{"success":true}') } - let(:http) { instance_double(Net::HTTP) } - - before do - allow(URI).to receive(:parse).and_return(uri) - allow(Net::HTTP::Get).to receive(:new).and_return(double('request')) - allow(Net::HTTP).to receive(:start).and_yield(http) - allow(http).to receive(:request).and_return(http_response) - allow(Oj).to receive(:load).with(http_response.body).and_return({ 'success' => true }) - end - - context 'with valid token format' do - it 'returns true when token is valid according to API' do - # Directly stub the regex validation - allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) - expect(token_service.validate_token(valid_token)).to be true - end - - it 'returns false when API returns success: false' do - # Directly stub the regex validation - allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) - allow(Oj).to receive(:load).with(http_response.body).and_return({ 'success' => false }) - expect(token_service.validate_token(valid_token)).to be false - end - - it 'handles network errors gracefully' do - # Directly stub the regex validation - allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) - allow(Net::HTTP).to receive(:start).and_raise(StandardError) - expect(token_service.validate_token(valid_token)).to be false - end - end - - context 'with invalid token format' do - it 'returns false for nil token' do - expect(token_service.validate_token(nil)).to be false - end - - it 'returns false for non-string token' do - expect(token_service.validate_token(123)).to be false - end - - it 'returns false for invalid format token' do - expect(token_service.validate_token(invalid_token)).to be false - end - end - end - - describe '#get_token' do - let(:config_file) { 'config.yml' } - - context 'when input token is present' do - it 'returns the input token' do - expect(token_service.get_token('input_token', 'file_token', config_file)).to eq('input_token') - end - end - - context 'when input token is blank but file token is present' do - it 'returns the file token' do - expect(token_service.get_token(nil, 'file_token', config_file)).to eq('file_token') - expect(token_service.get_token('', 'file_token', config_file)).to eq('file_token') - end - end - - context 'when both input and file tokens are blank' do - it 'prompts for token' do - expect(token_service).to receive(:prompt_for_token).with(config_file).and_return('new_token') - expect(token_service.get_token(nil, nil, config_file)).to eq('new_token') - end - end - end - - describe '#prompt_for_token' do - let(:config_file) { 'config.yml' } - let(:stdin_mock) { StringIO.new(valid_token) } - - before do - $stdout = StringIO.new - $stdin = stdin_mock - end - - after do - $stdout = STDOUT - $stdin = STDIN - end - - it 'prompts for token and returns it when valid' do - allow(token_service).to receive(:validate_token).with(valid_token).and_return(true) - expect(token_service.prompt_for_token(config_file)).to eq(valid_token) - end - - it 'continues prompting until valid token is entered' do - $stdin = StringIO.new("#{invalid_token}\n#{valid_token}") - allow(token_service).to receive(:validate_token).with(invalid_token).and_return(false) - allow(token_service).to receive(:validate_token).with(valid_token).and_return(true) - expect(token_service.prompt_for_token(config_file)).to eq(valid_token) - end - end - - describe '#fetch_upstream_repo' do - let(:uri) { URI.parse("#{submission_url}/submissions/resource?token=#{valid_token}") } - let(:http_response) { instance_double(Net::HTTPResponse, body: '{"repo_slug":"org/repo","spec_folder_sha":"abc123","source_code_url":"https://example.com/archive.zip"}') } - let(:http) { instance_double(Net::HTTP) } - let(:expected_result) { { 'repo_slug' => 'org/repo', 'spec_folder_sha' => 'abc123', 'source_code_url' => 'https://example.com/archive.zip' } } - - before do - allow(URI).to receive(:parse).and_return(uri) - allow(Net::HTTP::Get).to receive(:new).and_return(double('request')) - allow(Net::HTTP).to receive(:start).and_yield(http) - allow(http).to receive(:request).and_return(http_response) - allow(Oj).to receive(:load).with(http_response.body).and_return(expected_result) - end - - context 'with valid token format' do - it 'returns repository information hash' do - # Directly stub the regex validation - allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) - expect(token_service.fetch_upstream_repo(valid_token)).to eq(expected_result) - end - - it 'handles network errors gracefully' do - # Directly stub the regex validation - allow(valid_token).to receive(:=~).with(GradeRunner::Services::TokenService::TOKEN_REGEX).and_return(0) - allow(Net::HTTP).to receive(:start).and_raise(StandardError) - expect(token_service.fetch_upstream_repo(valid_token)).to be false - end - end - - context 'with invalid token format' do - it 'returns false' do - expect(token_service.fetch_upstream_repo(invalid_token)).to be false - end - end - end -end From a8d336d246e276bd585fa07fc72fe065ae8deff5 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Fri, 15 Aug 2025 15:28:16 -0500 Subject: [PATCH 19/34] Update README.markdown --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 959d9d3..e8b3b0c 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,6 @@ # grade_runner -A Ruby client for [Grades](https://grades.firstdraft.com) +A Ruby client for [Grades](https://grades.dpi.dev) ## Installation From 6dc2f18985e8fdf9a38e6e19854bd4e1c0287f72 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 18 Aug 2025 11:54:58 -0500 Subject: [PATCH 20/34] Upgrade oj to 3.16 to silence warning: ostruct was loaded from the standard library, but will no longer be part of the default gems starting from Ruby 3.5.0 --- Gemfile | 2 +- Gemfile.lock | 8 ++++++-- grade_runner.gemspec | 5 ++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index f5c3ce3..071cfe8 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source "https://rubygems.org" gem "activesupport", ">= 2.3.5" -gem "oj", "~> 3.13.12" +gem 'oj', '~> 3.16', '>= 3.16.3' gem "octokit", "~> 5.0" gem "zip" gem "faraday-retry", "~> 1.0.3" diff --git a/Gemfile.lock b/Gemfile.lock index 0f32768..7cd9825 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,6 +8,7 @@ GEM tzinfo (~> 2.0) addressable (2.8.4) public_suffix (>= 2.0.2, < 6.0) + bigdecimal (3.2.2) binding_of_caller (1.0.0) debug_inspector (>= 0.0.1) builder (3.2.4) @@ -89,7 +90,10 @@ GEM octokit (5.6.1) faraday (>= 1, < 3) sawyer (~> 0.9) - oj (3.13.23) + oj (3.16.11) + bigdecimal (>= 3.0) + ostruct (>= 0.2) + ostruct (0.6.3) pry (0.14.1) coderay (~> 1.1) method_source (~> 1.0) @@ -160,7 +164,7 @@ DEPENDENCIES faraday-retry (~> 1.0.3) juwelier (~> 2.4.9) octokit (~> 5.0) - oj (~> 3.13.12) + oj (~> 3.16, >= 3.16.3) pry (~> 0) pry-byebug (~> 3) pry-doc (~> 0) diff --git a/grade_runner.gemspec b/grade_runner.gemspec index a835d24..ea2cdfc 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -47,14 +47,14 @@ Gem::Specification.new do |s| ] s.homepage = "http://github.com/firstdraft/grade_runner".freeze s.licenses = ["MIT".freeze] - s.required_ruby_version = Gem::Requirement.new([">= 2".freeze, "< 3.4".freeze]) + s.required_ruby_version = Gem::Requirement.new([">= 2".freeze]) s.rubygems_version = "3.4.6".freeze s.summary = "A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)".freeze s.specification_version = 4 s.add_runtime_dependency(%q.freeze, [">= 2.3.5"]) - s.add_runtime_dependency(%q.freeze, ["~> 3.13.12"]) + s.add_runtime_dependency(%q.freeze, ["~> 3.16"]) s.add_runtime_dependency(%q.freeze, ["~> 5.0"]) s.add_runtime_dependency(%q.freeze, [">= 0"]) s.add_runtime_dependency(%q.freeze, ["~> 1.0.3"]) @@ -71,4 +71,3 @@ Gem::Specification.new do |s| s.add_development_dependency(%q.freeze, ["~> 1"]) s.add_development_dependency(%q.freeze, ["~> 0"]) end - From a217a05fb09623fddea0d606510e6b109be4f96e Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 18 Aug 2025 12:33:15 -0500 Subject: [PATCH 21/34] Refactor how taks are loaded. Now only need require 'grade_runner/tasks' in the Rakefile --- lib/grade_runner/railtie.rb | 6 +----- lib/grade_runner/tasks.rb | 8 ++++++++ lib/{ => grade_runner}/tasks/grade.rake | 0 lib/{ => grade_runner}/tasks/grade_runner.rake | 0 4 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 lib/grade_runner/tasks.rb rename lib/{ => grade_runner}/tasks/grade.rake (100%) rename lib/{ => grade_runner}/tasks/grade_runner.rake (100%) diff --git a/lib/grade_runner/railtie.rb b/lib/grade_runner/railtie.rb index 1151409..4b2f579 100644 --- a/lib/grade_runner/railtie.rb +++ b/lib/grade_runner/railtie.rb @@ -2,10 +2,6 @@ module GradeRunner class Railtie < ::Rails::Railtie - - rake_tasks do - load "tasks/grade_runner.rake" - load "tasks/grade.rake" - end + rake_tasks { load "grade_runner/tasks.rb" } end end diff --git a/lib/grade_runner/tasks.rb b/lib/grade_runner/tasks.rb new file mode 100644 index 0000000..5cbcd00 --- /dev/null +++ b/lib/grade_runner/tasks.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "rake" unless defined?(Rake) + +# Load any .rake files nested under lib/grade_runner/tasks/ +Dir[File.expand_path("tasks/**/*.rake", __dir__)].sort.each do |file| + load file +end diff --git a/lib/tasks/grade.rake b/lib/grade_runner/tasks/grade.rake similarity index 100% rename from lib/tasks/grade.rake rename to lib/grade_runner/tasks/grade.rake diff --git a/lib/tasks/grade_runner.rake b/lib/grade_runner/tasks/grade_runner.rake similarity index 100% rename from lib/tasks/grade_runner.rake rename to lib/grade_runner/tasks/grade_runner.rake From a5dbf9520ab5325280936231bd7a1cc32943c41f Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 18 Aug 2025 17:43:17 -0500 Subject: [PATCH 22/34] Fix issue with JsonOutputFormatter --- README.markdown | 1 - Rakefile | 2 +- grade_runner.gemspec | 16 ++++++++-------- .../formatters/json_output_formatter.rb | 3 ++- lib/grade_runner/services/spec_service.rb | 4 +++- lib/grade_runner/tasks/grade_runner.rake | 2 ++ 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/README.markdown b/README.markdown index e8b3b0c..f11dc0f 100644 --- a/README.markdown +++ b/README.markdown @@ -2,7 +2,6 @@ A Ruby client for [Grades](https://grades.dpi.dev) - ## Installation Add this line to your application's Gemfile: diff --git a/Rakefile b/Rakefile index fcca038..21a4bd4 100644 --- a/Rakefile +++ b/Rakefile @@ -22,7 +22,7 @@ Juwelier::Tasks.new do |gem| gem.authors = ["Raghu Betina", "Jelani Woods"] # Note: rspec tests do not yet support 3.4 (BigDecimal) - gem.required_ruby_version = Gem::Requirement.new(">= 2", "< 3.4") + gem.required_ruby_version = Gem::Requirement.new(">= 2") # dependencies defined in Gemfile end diff --git a/grade_runner.gemspec b/grade_runner.gemspec index ea2cdfc..153993e 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= s.require_paths = ["lib".freeze] s.authors = ["Raghu Betina".freeze, "Jelani Woods".freeze] - s.date = "2025-04-03" + s.date = "2025-08-18" s.description = "This gem runs your RSpec test suite and posts the JSON output to grades.firstdraft.com.".freeze s.email = ["raghu@firstdraft.com".freeze, "jelani@firstdraft.com".freeze] s.extra_rdoc_files = [ @@ -38,23 +38,22 @@ Gem::Specification.new do |s| "lib/grade_runner/services/grade_service.rb", "lib/grade_runner/services/spec_service.rb", "lib/grade_runner/services/token_service.rb", + "lib/grade_runner/tasks.rb", + "lib/grade_runner/tasks/grade.rake", + "lib/grade_runner/tasks/grade_runner.rake", "lib/grade_runner/utils/path_utils.rb", - "lib/tasks/grade.rake", - "lib/tasks/grade_runner.rake", - "spec/grade_service_spec.rb", - "spec/spec_helper.rb", - "spec/spec_service_spec.rb" + "spec/spec_helper.rb" ] s.homepage = "http://github.com/firstdraft/grade_runner".freeze s.licenses = ["MIT".freeze] - s.required_ruby_version = Gem::Requirement.new([">= 2".freeze]) + s.required_ruby_version = Gem::Requirement.new(">= 2".freeze) s.rubygems_version = "3.4.6".freeze s.summary = "A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)".freeze s.specification_version = 4 s.add_runtime_dependency(%q.freeze, [">= 2.3.5"]) - s.add_runtime_dependency(%q.freeze, ["~> 3.16"]) + s.add_runtime_dependency(%q.freeze, ["~> 3.16", ">= 3.16.3"]) s.add_runtime_dependency(%q.freeze, ["~> 5.0"]) s.add_runtime_dependency(%q.freeze, [">= 0"]) s.add_runtime_dependency(%q.freeze, ["~> 1.0.3"]) @@ -71,3 +70,4 @@ Gem::Specification.new do |s| s.add_development_dependency(%q.freeze, ["~> 1"]) s.add_development_dependency(%q.freeze, ["~> 0"]) end + diff --git a/lib/grade_runner/formatters/json_output_formatter.rb b/lib/grade_runner/formatters/json_output_formatter.rb index 86b5410..29ec1a7 100644 --- a/lib/grade_runner/formatters/json_output_formatter.rb +++ b/lib/grade_runner/formatters/json_output_formatter.rb @@ -1,5 +1,6 @@ -RSpec::Support.require_rspec_core "formatters/json_formatter" require "oj" +require "rspec/core" + class JsonOutputFormatter < RSpec::Core::Formatters::JsonFormatter RSpec::Core::Formatters.register self, :dump_summary diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb index a8c96c0..4bf2418 100644 --- a/lib/grade_runner/services/spec_service.rb +++ b/lib/grade_runner/services/spec_service.rb @@ -17,7 +17,9 @@ def run_tests(output_path) `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) # Run tests with JSON formatter - `RAILS_ENV=test bundle exec rspec --format JsonOutputFormatter --out #{output_path}` + # Run RSpec via its Ruby API so you stay in the same process + # This helps with using JsonOutputFormatter + RSpec::Core::Runner.run(["--format", "JsonOutputFormatter", "--out", output_path]) # Load and return test results Oj.load(File.read(output_path)) diff --git a/lib/grade_runner/tasks/grade_runner.rake b/lib/grade_runner/tasks/grade_runner.rake index 37b87d1..7436f12 100644 --- a/lib/grade_runner/tasks/grade_runner.rake +++ b/lib/grade_runner/tasks/grade_runner.rake @@ -1,5 +1,7 @@ require "grade_runner/services/grade_service" require "grade_runner/utils/path_utils" +require "grade_runner/formatters/json_output_formatter" +require "grade_runner/formatters/hint_formatter" namespace :grade_runner do desc "Grade project" From e2a54db64fc71373e0151543bb533af1d7c55b93 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Tue, 19 Aug 2025 16:19:03 -0500 Subject: [PATCH 23/34] Explicitly set spec directory so rspec can find tests to run. --- lib/grade_runner/services/spec_service.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb index 4bf2418..7aed4f0 100644 --- a/lib/grade_runner/services/spec_service.rb +++ b/lib/grade_runner/services/spec_service.rb @@ -16,10 +16,21 @@ def run_tests(output_path) # Ensure database is migrated if Rails app `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - # Run tests with JSON formatter + # Run tests with JSON formatter and explicitly set path to specs + project_root = Dir.pwd + spec_dir = File.join(project_root, "spec") + args = [ + "--default-path", spec_dir, + "--pattern", "**/*_spec.rb", + "--format", "JsonOutputFormatter", + "--out", output_path, # "tmp/output/results.json" + spec_dir + ] + # Run RSpec via its Ruby API so you stay in the same process # This helps with using JsonOutputFormatter - RSpec::Core::Runner.run(["--format", "JsonOutputFormatter", "--out", output_path]) + # TODO: handle non-zero exit code + RSpec::Core::Runner.run(args) # Load and return test results Oj.load(File.read(output_path)) From 2741c2621a47c43d3d5e7b8b728cc9ddad88d8df Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 13:51:31 -0500 Subject: [PATCH 24/34] Add test helpers for rspec tests using grade_runner --- lib/grade_runner/test_helpers.rb | 133 +++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 lib/grade_runner/test_helpers.rb diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb new file mode 100644 index 0000000..a1d1019 --- /dev/null +++ b/lib/grade_runner/test_helpers.rb @@ -0,0 +1,133 @@ +# lib/grade_runner/test_helpers.rb +require "stringio" + +module GradeRunner + ## + # TestHelpers provides small utilities for writing exercise specs + # that need to capture program output or temporarily modify the + # runtime environment. These helpers are designed to make specs + # shorter, clearer, and more consistent across projects. + # + # Example usage (RSpec): + # + # RSpec.configure do |config| + # config.include GradeRunner::TestHelpers + # end + # + # it "captures script output" do + # output = capture_output_of("./hello.rb") + # expect(output).to eq("\"hello, world\"\n") + # end + # + module TestHelpers + ## + # Capture standard output (and optionally standard error) while + # executing a block. + # + # @param capture_stderr [Boolean] whether to also capture $stderr + # @yield block of code to execute + # @return [String] captured STDOUT if capture_stderr: false + # @return [Array] [STDOUT, STDERR] if capture_stderr: true + # + # @example + # output = capture_stdout { puts "hi" } + # # => "hi\n" + # + # out, err = capture_stdout(capture_stderr: true) do + # puts "ok" + # warn "oops" + # end + # # out = "ok\n" + # # err = "oops\n" + # + def capture_stdout(capture_stderr: false) + orig_out, orig_err = $stdout, $stderr + out_buf = StringIO.new + err_buf = capture_stderr ? StringIO.new : nil + + $stdout = out_buf + $stderr = err_buf if capture_stderr + + yield + + capture_stderr ? [out_buf.string, err_buf.string] : out_buf.string + ensure + $stdout = orig_out + $stderr = orig_err if capture_stderr + end + + ## + # Convenience alias for capture_stdout with capture_stderr: true. + # + # @yield block of code to execute + # @return [Array] [STDOUT, STDERR] + # + # @example + # out, err = capture_io { puts "ok"; warn "oops" } + # + def capture_io(&block) + capture_stdout(capture_stderr: true, &block) + end + + ## + # Load a Ruby file (like "ruby file.rb") and capture its output. + # + # This is especially useful for testing beginner exercises + # where the code lives in a standalone script and prints with pp/puts. + # + # @param file_path [String] path to the file to load + # @param capture_stderr [Boolean] whether to also capture $stderr + # @return [String, Array] captured output + # + # @example + # output = capture_output_of("./hello.rb") + # expect(output).to eq("\"hello, world\"\n") + # + def capture_output_of(file_path, capture_stderr: false) + capture_stdout(capture_stderr: capture_stderr) { load file_path } + end + + ## + # Temporarily set environment variables within a block. + # + # Original values are restored afterward, even if the block raises. + # + # @param hash [Hash{String,Symbol=>String,nil}] ENV vars to override + # @yield block to run with modified ENV + # + # @example + # with_env("LUCKY" => "14") do + # output = capture_output_of("./lucky_number.rb") + # expect(output).to include("14") + # end + # + def with_env(hash) + old = {} + hash.each { |k, v| old[k] = ENV[k]; ENV[k] = v } + yield + ensure + old.each { |k, v| ENV[k] = v } + end + + ## + # Temporarily change the current working directory for a block. + # + # Useful when testing scripts that assume a particular relative path. + # + # @param dir [String] directory to change into + # @yield block to run inside the directory + # + # @example + # with_chdir("examples") do + # output = capture_output_of("./script.rb") + # end + # + def with_chdir(dir) + old = Dir.pwd + Dir.chdir(dir) + yield + ensure + Dir.chdir(old) + end + end +end From 78c468400d42f7fdc5a32ec236f6ccf4cb13708c Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 13:56:27 -0500 Subject: [PATCH 25/34] Add test_helpers to module --- lib/grade_runner.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/grade_runner.rb b/lib/grade_runner.rb index b77eab8..8466eb4 100644 --- a/lib/grade_runner.rb +++ b/lib/grade_runner.rb @@ -6,6 +6,7 @@ require "grade_runner/services/spec_service" require "grade_runner/services/grade_service" require "grade_runner/railtie" if defined?(Rails) +require "grade_runner/test_helpers" module GradeRunner class Error < StandardError; end From 67eb5ea5228f5faaad9ad62a1a3c76e8997204f6 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 14:04:59 -0500 Subject: [PATCH 26/34] Remove test helpers from default imports --- lib/grade_runner.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/grade_runner.rb b/lib/grade_runner.rb index 8466eb4..b77eab8 100644 --- a/lib/grade_runner.rb +++ b/lib/grade_runner.rb @@ -6,7 +6,6 @@ require "grade_runner/services/spec_service" require "grade_runner/services/grade_service" require "grade_runner/railtie" if defined?(Rails) -require "grade_runner/test_helpers" module GradeRunner class Error < StandardError; end From 0e3c579ad5796e02f1758bcf97421b7772a9d32a Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 14:45:18 -0500 Subject: [PATCH 27/34] Refactor some test helper methods --- lib/grade_runner/test_helpers.rb | 103 ++++++++++++++----------------- 1 file changed, 46 insertions(+), 57 deletions(-) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index a1d1019..90077c0 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -4,9 +4,8 @@ module GradeRunner ## # TestHelpers provides small utilities for writing exercise specs - # that need to capture program output or temporarily modify the - # runtime environment. These helpers are designed to make specs - # shorter, clearer, and more consistent across projects. + # that need to capture program output. These helpers keep specs + # short, readable, and consistent across projects. # # Example usage (RSpec): # @@ -15,8 +14,8 @@ module GradeRunner # end # # it "captures script output" do - # output = capture_output_of("./hello.rb") - # expect(output).to eq("\"hello, world\"\n") + # lines = pp_lines_from("./hello.rb") + # expect(lines).to eq(["hello, world"]) # end # module TestHelpers @@ -30,15 +29,7 @@ module TestHelpers # @return [Array] [STDOUT, STDERR] if capture_stderr: true # # @example - # output = capture_stdout { puts "hi" } - # # => "hi\n" - # - # out, err = capture_stdout(capture_stderr: true) do - # puts "ok" - # warn "oops" - # end - # # out = "ok\n" - # # err = "oops\n" + # output = capture_stdout { puts "hi" } # => "hi\n" # def capture_stdout(capture_stderr: false) orig_out, orig_err = $stdout, $stderr @@ -57,20 +48,7 @@ def capture_stdout(capture_stderr: false) end ## - # Convenience alias for capture_stdout with capture_stderr: true. - # - # @yield block of code to execute - # @return [Array] [STDOUT, STDERR] - # - # @example - # out, err = capture_io { puts "ok"; warn "oops" } - # - def capture_io(&block) - capture_stdout(capture_stderr: true, &block) - end - - ## - # Load a Ruby file (like "ruby file.rb") and capture its output. + # Load a Ruby file (like `ruby file.rb`) and capture its output. # # This is especially useful for testing beginner exercises # where the code lives in a standalone script and prints with pp/puts. @@ -88,46 +66,57 @@ def capture_output_of(file_path, capture_stderr: false) end ## - # Temporarily set environment variables within a block. + # Shorthand to run a script in the current process and return its STDOUT. # - # Original values are restored afterward, even if the block raises. + # @param path [String] the script path (e.g., "./script.rb") + # @return [String] captured stdout # - # @param hash [Hash{String,Symbol=>String,nil}] ENV vars to override - # @yield block to run with modified ENV + # @example + # out = run_script("./hello.rb") + # + def run_script(path) + capture_output_of(path) + end + + ## + # Normalize output produced via pp/puts by: + # - removing double quotes (pp wraps strings in quotes), + # - splitting into lines, + # - trimming whitespace, + # - removing empty lines. + # + # @param output [String] + # @return [Array] cleaned lines # # @example - # with_env("LUCKY" => "14") do - # output = capture_output_of("./lucky_number.rb") - # expect(output).to include("14") - # end - # - def with_env(hash) - old = {} - hash.each { |k, v| old[k] = ENV[k]; ENV[k] = v } - yield - ensure - old.each { |k, v| ENV[k] = v } + # lines = normalize_output(%("14"\n"lucky"\n)) # => ["14", "lucky"] + # + def normalize_output(output) + output.gsub('"', '').lines.map(&:strip).reject(&:empty?) end ## - # Temporarily change the current working directory for a block. + # Convenience for specs where scripts print with `pp`: + # Runs the file and returns normalized lines (see #normalize_output). # - # Useful when testing scripts that assume a particular relative path. + # If rand_value is provided, stubs bare `rand(...)` calls by + # intercepting Kernel#rand as invoked on Object instances + # (i.e., `rand(1..100)` with no explicit receiver). # - # @param dir [String] directory to change into - # @yield block to run inside the directory + # @param file_path [String] + # @param rand_value [Integer, nil] optional stub value for rand + # @return [Array] normalized lines # # @example - # with_chdir("examples") do - # output = capture_output_of("./script.rb") - # end - # - def with_chdir(dir) - old = Dir.pwd - Dir.chdir(dir) - yield - ensure - Dir.chdir(old) + # lines = pp_lines_from("./lucky_number.rb", rand_value: 14) + # expect(lines).to eq(%w[14 lucky]) + # + def pp_lines_from(file_path, rand_value: nil) + if rand_value + # Requires RSpec mocks (this module is intended to be included in RSpec) + allow_any_instance_of(Object).to receive(:rand).and_return(rand_value) + end + normalize_output(run_script(file_path)) end end end From 6ecefa31c9a2b5184654c6b51b3138dd6a8c796c Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 15:05:06 -0500 Subject: [PATCH 28/34] Add helpers to run scripts with stdin --- lib/grade_runner/test_helpers.rb | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index 90077c0..f2e0e49 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -118,5 +118,40 @@ def pp_lines_from(file_path, rand_value: nil) end normalize_output(run_script(file_path)) end + + ## + # Temporarily replace $stdin with a StringIO seeded with given input. + # + # @param input [String] text that will be returned by gets/lines/etc. + # @yield block that will run with $stdin replaced + # @return [Object] block return value + # + # @example + # with_stdin("7\n3\n") do + # x = gets.to_i # => 7 + # y = gets.to_i # => 3 + # end + # + def with_stdin(input) + original_stdin = $stdin + $stdin = StringIO.new(input) + yield + ensure + $stdin = original_stdin + end + + ## + # Run a Ruby script with provided stdin content, capturing its stdout. + # + # @param path [String] path to the script (e.g., "./calculator.rb") + # @param input [String] text passed to $stdin + # @return [String] captured stdout + # + # @example + # output = run_script_with_input("./calculator.rb", "7\n3\n") + # + def run_script_with_input(path, input) + with_stdin(input) { run_script(path) } + end end end From b2bbd22a4958b23479a822d13575e5ae38863472 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 15:10:39 -0500 Subject: [PATCH 29/34] Add helper method to strip comments from source --- lib/grade_runner/test_helpers.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index f2e0e49..3354cec 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -153,5 +153,28 @@ def with_stdin(input) def run_script_with_input(path, input) with_stdin(input) { run_script(path) } end + + ## + # Remove comment-only lines from a source string. + # + # This is useful in specs when you want to assert that a script + # uses certain operators (`+`, `-`, `%`, etc.) without matching + # against instructional comments. + # + # @param src [String] the file contents + # @return [String] source with comment-only lines removed + # + # @example + # src = <<~RUBY + # # add two numbers + # x + y + # RUBY + # + # strip_comments(src) + # # => "x + y\n" + # + def strip_comments(src) + src.lines.reject { |line| line.strip.start_with?("#") }.join + end end end From c239f8e0c757750f16e3ac07afb9200daf82f9fc Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 15:59:56 -0500 Subject: [PATCH 30/34] Restructure test helper methods --- lib/grade_runner/test_helpers.rb | 196 +++++++++++++------------------ 1 file changed, 83 insertions(+), 113 deletions(-) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index 3354cec..1458c07 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -14,167 +14,137 @@ module GradeRunner # end # # it "captures script output" do - # lines = pp_lines_from("./hello.rb") + # lines = pp_lines_from_run("./hello.rb") # expect(lines).to eq(["hello, world"]) # end # module TestHelpers + Status = Struct.new(:exitstatus) do + def success? = exitstatus.to_i == 0 + end + ## - # Capture standard output (and optionally standard error) while - # executing a block. + # Run a Ruby script in-process (like `ruby file.rb`) while capturing + # its stdout, stderr, and exitstatus. This is similar to + # Open3.capture3 but plays nicely with RSpec mocks. # - # @param capture_stderr [Boolean] whether to also capture $stderr - # @yield block of code to execute - # @return [String] captured STDOUT if capture_stderr: false - # @return [Array] [STDOUT, STDERR] if capture_stderr: true + # @param path [String] path to the script (e.g., "./calculator.rb") + # @param stdin [String] content fed into gets/$stdin (include "\n") + # @param argv [Array] values to replace ARGV with + # @return [Array(String, String, Status)] [stdout, stderr, status] # # @example - # output = capture_stdout { puts "hi" } # => "hi\n" + # out, err, status = run_script("./hello.rb", stdin: "7\n3\n") # - def capture_stdout(capture_stderr: false) - orig_out, orig_err = $stdout, $stderr + def run_script(path, stdin: "", argv: []) + orig_stdin, orig_stdout, orig_stderr = $stdin, $stdout, $stderr + orig_argv = ARGV.dup + + in_buf = StringIO.new(String(stdin)) out_buf = StringIO.new - err_buf = capture_stderr ? StringIO.new : nil + err_buf = StringIO.new + $stdin = in_buf $stdout = out_buf - $stderr = err_buf if capture_stderr + $stderr = err_buf + ARGV.replace(Array(argv)) - yield + status = Status.new(0) - capture_stderr ? [out_buf.string, err_buf.string] : out_buf.string - ensure - $stdout = orig_out - $stderr = orig_err if capture_stderr - end + begin + load path + rescue SystemExit => e + status.exitstatus = e.status || 1 + rescue Exception => e + err_buf.puts("#{e.class}: #{e.message}") + e.backtrace&.each { |ln| err_buf.puts(ln) } + status.exitstatus = 1 + ensure + $stdin = orig_stdin + $stdout = orig_stdout + $stderr = orig_stderr + ARGV.replace(orig_argv) + end - ## - # Load a Ruby file (like `ruby file.rb`) and capture its output. - # - # This is especially useful for testing beginner exercises - # where the code lives in a standalone script and prints with pp/puts. - # - # @param file_path [String] path to the file to load - # @param capture_stderr [Boolean] whether to also capture $stderr - # @return [String, Array] captured output - # - # @example - # output = capture_output_of("./hello.rb") - # expect(output).to eq("\"hello, world\"\n") - # - def capture_output_of(file_path, capture_stderr: false) - capture_stdout(capture_stderr: capture_stderr) { load file_path } + [out_buf.string, err_buf.string, status] end + alias capture3_ruby run_script ## - # Shorthand to run a script in the current process and return its STDOUT. + # Run a script and return normalized pp/puts lines: + # - strips quotes (pp wraps strings in quotes) + # - trims whitespace + # - drops empty lines # - # @param path [String] the script path (e.g., "./script.rb") - # @return [String] captured stdout + # @param path [String] + # @param stdin [String] + # @return [Array] # # @example - # out = run_script("./hello.rb") + # lines = pp_lines_from_run("./lucky_number.rb", stdin: "14\n") + # expect(lines).to eq(%w[14 lucky]) # - def run_script(path) - capture_output_of(path) + def pp_lines_from_run(path, stdin: "") + stdout, _stderr, _status = run_script(path, stdin: stdin) + normalize_output(stdout) end + alias pp_lines_from_capture3 pp_lines_from_run ## - # Normalize output produced via pp/puts by: - # - removing double quotes (pp wraps strings in quotes), - # - splitting into lines, - # - trimming whitespace, - # - removing empty lines. + # Remove quotes, split into lines, strip, and reject empties. # # @param output [String] - # @return [Array] cleaned lines - # - # @example - # lines = normalize_output(%("14"\n"lucky"\n)) # => ["14", "lucky"] - # + # @return [Array] def normalize_output(output) - output.gsub('"', '').lines.map(&:strip).reject(&:empty?) + output.gsub('"', "").lines.map(&:strip).reject(&:empty?) end ## - # Convenience for specs where scripts print with `pp`: - # Runs the file and returns normalized lines (see #normalize_output). - # - # If rand_value is provided, stubs bare `rand(...)` calls by - # intercepting Kernel#rand as invoked on Object instances - # (i.e., `rand(1..100)` with no explicit receiver). - # - # @param file_path [String] - # @param rand_value [Integer, nil] optional stub value for rand - # @return [Array] normalized lines + # Run a Ruby file and capture its stdout. # - # @example - # lines = pp_lines_from("./lucky_number.rb", rand_value: 14) - # expect(lines).to eq(%w[14 lucky]) + # @param path [String] + # @return [String] # - def pp_lines_from(file_path, rand_value: nil) - if rand_value - # Requires RSpec mocks (this module is intended to be included in RSpec) - allow_any_instance_of(Object).to receive(:rand).and_return(rand_value) - end - normalize_output(run_script(file_path)) + def run_file(path) + capture_stdout { load path } end + alias run_script run_file ## - # Temporarily replace $stdin with a StringIO seeded with given input. + # Capture standard output (and optionally stderr) while running a block. # - # @param input [String] text that will be returned by gets/lines/etc. - # @yield block that will run with $stdin replaced - # @return [Object] block return value + # @param capture_stderr [Boolean] + # @yield block to run + # @return [String] stdout, or [stdout, stderr] if capture_stderr # - # @example - # with_stdin("7\n3\n") do - # x = gets.to_i # => 7 - # y = gets.to_i # => 3 - # end - # - def with_stdin(input) - original_stdin = $stdin - $stdin = StringIO.new(input) + def capture_stdout(capture_stderr: false) + orig_out, orig_err = $stdout, $stderr + out_buf = StringIO.new + err_buf = capture_stderr ? StringIO.new : nil + + $stdout = out_buf + $stderr = err_buf if capture_stderr + yield - ensure - $stdin = original_stdin - end - ## - # Run a Ruby script with provided stdin content, capturing its stdout. - # - # @param path [String] path to the script (e.g., "./calculator.rb") - # @param input [String] text passed to $stdin - # @return [String] captured stdout - # - # @example - # output = run_script_with_input("./calculator.rb", "7\n3\n") - # - def run_script_with_input(path, input) - with_stdin(input) { run_script(path) } + capture_stderr ? [out_buf.string, err_buf.string] : out_buf.string + ensure + $stdout = orig_out + $stderr = orig_err if capture_stderr end ## - # Remove comment-only lines from a source string. - # - # This is useful in specs when you want to assert that a script - # uses certain operators (`+`, `-`, `%`, etc.) without matching - # against instructional comments. + # Remove comment-only lines from source. # - # @param src [String] the file contents - # @return [String] source with comment-only lines removed + # @param src [String] + # @return [String] source without comments # # @example - # src = <<~RUBY - # # add two numbers - # x + y - # RUBY - # - # strip_comments(src) - # # => "x + y\n" + # clean = source_without_comments(File.read("calculator.rb")) # - def strip_comments(src) + def source_without_comments(src) src.lines.reject { |line| line.strip.start_with?("#") }.join end + alias strip_comments source_without_comments end end From 15682a0d74201aa1c6d3fd3292c42cf18aecd3ae Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 16:16:19 -0500 Subject: [PATCH 31/34] Fix name conflict with run_script --- lib/grade_runner/test_helpers.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index 1458c07..c0f48cd 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -25,8 +25,7 @@ def success? = exitstatus.to_i == 0 ## # Run a Ruby script in-process (like `ruby file.rb`) while capturing - # its stdout, stderr, and exitstatus. This is similar to - # Open3.capture3 but plays nicely with RSpec mocks. + # its stdout, stderr, and exitstatus. # # @param path [String] path to the script (e.g., "./calculator.rb") # @param stdin [String] content fed into gets/$stdin (include "\n") @@ -68,7 +67,6 @@ def run_script(path, stdin: "", argv: []) [out_buf.string, err_buf.string, status] end - alias capture3_ruby run_script ## # Run a script and return normalized pp/puts lines: @@ -108,7 +106,6 @@ def normalize_output(output) def run_file(path) capture_stdout { load path } end - alias run_script run_file ## # Capture standard output (and optionally stderr) while running a block. From ad6b02ed817081837fbbb914f718e64c3e504378 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Mon, 8 Sep 2025 16:53:58 -0500 Subject: [PATCH 32/34] Remove alias --- lib/grade_runner/test_helpers.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index c0f48cd..74909f8 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -86,7 +86,6 @@ def pp_lines_from_run(path, stdin: "") stdout, _stderr, _status = run_script(path, stdin: stdin) normalize_output(stdout) end - alias pp_lines_from_capture3 pp_lines_from_run ## # Remove quotes, split into lines, strip, and reject empties. From 4cc94426256c0a57d662369fdd3824e2d297d164 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Wed, 10 Sep 2025 11:02:55 -0500 Subject: [PATCH 33/34] Refactor test helper method names --- lib/grade_runner/test_helpers.rb | 105 +++++++++++-------------------- 1 file changed, 36 insertions(+), 69 deletions(-) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index 74909f8..b685766 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -9,12 +9,19 @@ module GradeRunner # # Example usage (RSpec): # + # # spec/spec_helper.rb + # + # require "bundler/setup" # !important so the Gemfile dependencies are on $LOAD_PATH + # require "grade_runner/test_helpers" + # # RSpec.configure do |config| # config.include GradeRunner::TestHelpers # end # + # # spec/example_spec.rb + # # it "captures script output" do - # lines = pp_lines_from_run("./hello.rb") + # lines = run_ruby_lines("./hello.rb") # expect(lines).to eq(["hello, world"]) # end # @@ -23,18 +30,13 @@ module TestHelpers def success? = exitstatus.to_i == 0 end - ## - # Run a Ruby script in-process (like `ruby file.rb`) while capturing - # its stdout, stderr, and exitstatus. + # Run a Ruby script (in-process) and capture stdout, stderr, and status. + # Works with RSpec stubs because no subprocess is spawned. # - # @param path [String] path to the script (e.g., "./calculator.rb") - # @param stdin [String] content fed into gets/$stdin (include "\n") + # @param path [String] e.g. "./calculator.rb" + # @param stdin [String] content for gets/$stdin (include trailing "\n"s) # @param argv [Array] values to replace ARGV with # @return [Array(String, String, Status)] [stdout, stderr, status] - # - # @example - # out, err, status = run_script("./hello.rb", stdin: "7\n3\n") - # def run_script(path, stdin: "", argv: []) orig_stdin, orig_stdout, orig_stderr = $stdin, $stdout, $stderr orig_argv = ARGV.dup @@ -67,80 +69,45 @@ def run_script(path, stdin: "", argv: []) [out_buf.string, err_buf.string, status] end + alias run_ruby run_script - ## - # Run a script and return normalized pp/puts lines: - # - strips quotes (pp wraps strings in quotes) - # - trims whitespace - # - drops empty lines + # Run a script and return cleaned lines (quotes stripped, trimmed, no blanks). # - # @param path [String] - # @param stdin [String] # @return [Array] + def run_script_and_capture_lines(path, stdin: "") + stdout, _stderr, _status = run_ruby(path, stdin: stdin) + clean_output_lines(stdout) + end + alias run_ruby_and_capture_lines run_script_and_capture_lines + + # Run a script and return raw stdout (string). # - # @example - # lines = pp_lines_from_run("./lucky_number.rb", stdin: "14\n") - # expect(lines).to eq(%w[14 lucky]) - # - def pp_lines_from_run(path, stdin: "") - stdout, _stderr, _status = run_script(path, stdin: stdin) - normalize_output(stdout) + # @return [String] + def capture_raw_stdout_from(path, stdin: "") + stdout, _stderr, _status = run_ruby(path, stdin: stdin) + stdout end - ## - # Remove quotes, split into lines, strip, and reject empties. + # Normalize printed output into human-friendly lines: + # - remove pp quotes + # - split lines + # - strip whitespace + # - drop empties # # @param output [String] # @return [Array] def normalize_output(output) output.gsub('"', "").lines.map(&:strip).reject(&:empty?) end + alias clean_output_lines normalize_output - ## - # Run a Ruby file and capture its stdout. + # Remove comment-only lines from source text. # - # @param path [String] + # @param source [String] # @return [String] - # - def run_file(path) - capture_stdout { load path } - end - - ## - # Capture standard output (and optionally stderr) while running a block. - # - # @param capture_stderr [Boolean] - # @yield block to run - # @return [String] stdout, or [stdout, stderr] if capture_stderr - # - def capture_stdout(capture_stderr: false) - orig_out, orig_err = $stdout, $stderr - out_buf = StringIO.new - err_buf = capture_stderr ? StringIO.new : nil - - $stdout = out_buf - $stderr = err_buf if capture_stderr - - yield - - capture_stderr ? [out_buf.string, err_buf.string] : out_buf.string - ensure - $stdout = orig_out - $stderr = orig_err if capture_stderr - end - - ## - # Remove comment-only lines from source. - # - # @param src [String] - # @return [String] source without comments - # - # @example - # clean = source_without_comments(File.read("calculator.rb")) - # - def source_without_comments(src) - src.lines.reject { |line| line.strip.start_with?("#") }.join + def strip_comments(source) + source.lines.reject { |line| line.strip.start_with?("#") }.join end - alias strip_comments source_without_comments + alias strip_comment_lines strip_comments end end From 45de32c2909659f55ac222736cb698c424739598 Mon Sep 17 00:00:00 2001 From: Ian Heraty Date: Thu, 11 Sep 2025 10:55:44 -0500 Subject: [PATCH 34/34] Add more support for argv --- lib/grade_runner/test_helpers.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb index b685766..70db525 100644 --- a/lib/grade_runner/test_helpers.rb +++ b/lib/grade_runner/test_helpers.rb @@ -74,8 +74,8 @@ def run_script(path, stdin: "", argv: []) # Run a script and return cleaned lines (quotes stripped, trimmed, no blanks). # # @return [Array] - def run_script_and_capture_lines(path, stdin: "") - stdout, _stderr, _status = run_ruby(path, stdin: stdin) + def run_script_and_capture_lines(path, stdin: "", argv: []) + stdout, _stderr, _status = run_ruby(path, stdin:, argv:) clean_output_lines(stdout) end alias run_ruby_and_capture_lines run_script_and_capture_lines @@ -83,8 +83,8 @@ def run_script_and_capture_lines(path, stdin: "") # Run a script and return raw stdout (string). # # @return [String] - def capture_raw_stdout_from(path, stdin: "") - stdout, _stderr, _status = run_ruby(path, stdin: stdin) + def capture_raw_stdout_from(path, stdin: "", argv: []) + stdout, _stderr, _status = run_ruby(path, stdin:, argv:) stdout end