diff --git a/Gemfile b/Gemfile index dc2b96c..071cfe8 100644 --- a/Gemfile +++ b/Gemfile @@ -1,19 +1,17 @@ source "https://rubygems.org" -# Add dependencies required to use your gem here. -# Example: + gem "activesupport", ">= 2.3.5" -gem "oj", "~> 3.13.12" +gem 'oj', '~> 3.16', '>= 3.16.3' gem "octokit", "~> 5.0" gem "zip" gem "faraday-retry", "~> 1.0.3" gem "rake", "~> 13" -# Add dependencies to develop your gem here. -# Include everything needed to run rake, tests, features, etc. + group :development do gem "rspec", "~> 3.5.0" gem "rdoc", "~> 6.1" - gem "bundler", "~> 2.1.4" - gem "juwelier", "~> 2.1.0" + gem "bundler", "~> 2.6.7" + gem "juwelier", "~> 2.4.9" gem "simplecov", ">= 0" gem "pry", "~> 0" gem "pry-byebug", "~> 3" diff --git a/Gemfile.lock b/Gemfile.lock index e3877ff..7cd9825 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,6 +8,7 @@ GEM tzinfo (~> 2.0) addressable (2.8.4) public_suffix (>= 2.0.2, < 6.0) + bigdecimal (3.2.2) binding_of_caller (1.0.0) debug_inspector (>= 0.0.1) builder (3.2.4) @@ -56,17 +57,21 @@ GEM i18n (1.14.1) concurrent-ruby (~> 1.0) interception (0.5) - juwelier (2.1.3) + juwelier (2.4.9) builder - bundler (>= 1.13) - git (>= 1.2.5) + bundler + git github_api - highline (>= 1.6.15) - nokogiri (>= 1.5.10) + highline + kamelcase (~> 0) + nokogiri + psych rake rdoc - semver + semver2 jwt (2.7.0) + kamelcase (0.0.2) + semver2 (~> 3) method_source (1.0.0) mini_portile2 (2.8.1) minitest (5.19.0) @@ -85,7 +90,10 @@ GEM octokit (5.6.1) faraday (>= 1, < 3) sawyer (~> 0.9) - oj (3.13.23) + oj (3.16.11) + bigdecimal (>= 3.0) + ostruct (>= 0.2) + ostruct (0.6.3) pry (0.14.1) coderay (~> 1.1) method_source (~> 1.0) @@ -130,7 +138,7 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) - semver (1.0.1) + semver2 (3.4.2) simplecov (0.21.2) docile (~> 1.1) simplecov-html (~> 0.11) @@ -152,11 +160,11 @@ PLATFORMS DEPENDENCIES activesupport (>= 2.3.5) - bundler (~> 2.1.4) + bundler (~> 2.6.7) faraday-retry (~> 1.0.3) - juwelier (~> 2.1.0) + juwelier (~> 2.4.9) octokit (~> 5.0) - oj (~> 3.13.12) + oj (~> 3.16, >= 3.16.3) pry (~> 0) pry-byebug (~> 3) pry-doc (~> 0) @@ -170,4 +178,4 @@ DEPENDENCIES zip BUNDLED WITH - 2.1.4 + 2.6.7 diff --git a/README.markdown b/README.markdown index f19897c..f11dc0f 100644 --- a/README.markdown +++ b/README.markdown @@ -1,7 +1,6 @@ # grade_runner -A Ruby client for [firstdraft Grades](https://grades.firstdraft.com) - +A Ruby client for [Grades](https://grades.dpi.dev) ## Installation @@ -48,6 +47,7 @@ if Rails.env.development? || Rails.env.test? GradeRunner.config do |config| config.default_points = 1 # default 1 config.override_local_specs = false # default true + config.submission_url = "https://your-grading-server-url.com" # Set custom submission URL end end ``` diff --git a/Rakefile b/Rakefile index 1d612ce..21a4bd4 100644 --- a/Rakefile +++ b/Rakefile @@ -21,6 +21,9 @@ Juwelier::Tasks.new do |gem| gem.email = ["raghu@firstdraft.com", "jelani@firstdraft.com"] gem.authors = ["Raghu Betina", "Jelani Woods"] + # Note: rspec tests do not yet support 3.4 (BigDecimal) + gem.required_ruby_version = Gem::Requirement.new(">= 2") + # dependencies defined in Gemfile end Juwelier::RubygemsDotOrgTasks.new diff --git a/VERSION b/VERSION index 43b2961..9789c4c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.13 +0.0.14 diff --git a/grade_runner.gemspec b/grade_runner.gemspec index 0c171a9..153993e 100644 --- a/grade_runner.gemspec +++ b/grade_runner.gemspec @@ -2,16 +2,16 @@ # DO NOT EDIT THIS FILE DIRECTLY # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec' # -*- encoding: utf-8 -*- -# stub: grade_runner 0.0.13 ruby lib +# stub: grade_runner 0.0.14 ruby lib Gem::Specification.new do |s| s.name = "grade_runner".freeze - s.version = "0.0.13" + s.version = "0.0.14" s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= s.require_paths = ["lib".freeze] s.authors = ["Raghu Betina".freeze, "Jelani Woods".freeze] - s.date = "2025-03-17" + s.date = "2025-08-18" s.description = "This gem runs your RSpec test suite and posts the JSON output to grades.firstdraft.com.".freeze s.email = ["raghu@firstdraft.com".freeze, "jelani@firstdraft.com".freeze] s.extra_rdoc_files = [ @@ -29,57 +29,45 @@ Gem::Specification.new do |s| "VERSION", "grade_runner.gemspec", "lib/grade_runner.rb", + "lib/grade_runner/formatters/hint_formatter.rb", + "lib/grade_runner/formatters/json_output_formatter.rb", "lib/grade_runner/railtie.rb", "lib/grade_runner/runner.rb", - "lib/tasks/grade.rake", - "lib/tasks/grade_runner.rake", + "lib/grade_runner/services/config_service.rb", + "lib/grade_runner/services/github_service.rb", + "lib/grade_runner/services/grade_service.rb", + "lib/grade_runner/services/spec_service.rb", + "lib/grade_runner/services/token_service.rb", + "lib/grade_runner/tasks.rb", + "lib/grade_runner/tasks/grade.rake", + "lib/grade_runner/tasks/grade_runner.rake", + "lib/grade_runner/utils/path_utils.rb", "spec/spec_helper.rb" ] s.homepage = "http://github.com/firstdraft/grade_runner".freeze s.licenses = ["MIT".freeze] - s.rubygems_version = "3.1.6".freeze + s.required_ruby_version = Gem::Requirement.new(">= 2".freeze) + s.rubygems_version = "3.4.6".freeze s.summary = "A Ruby client for [firstdraft Grades](https://grades.firstdraft.com)".freeze - if s.respond_to? :specification_version then - s.specification_version = 4 - end + s.specification_version = 4 - if s.respond_to? :add_runtime_dependency then - s.add_runtime_dependency(%q.freeze, [">= 2.3.5"]) - s.add_runtime_dependency(%q.freeze, ["~> 3.13.12"]) - s.add_runtime_dependency(%q.freeze, ["~> 5.0"]) - s.add_runtime_dependency(%q.freeze, [">= 0"]) - s.add_runtime_dependency(%q.freeze, ["~> 1.0.3"]) - s.add_runtime_dependency(%q.freeze, ["~> 13"]) - s.add_development_dependency(%q.freeze, ["~> 3.5.0"]) - s.add_development_dependency(%q.freeze, ["~> 6.1"]) - s.add_development_dependency(%q.freeze, ["~> 2.1.4"]) - s.add_development_dependency(%q.freeze, ["~> 2.1.0"]) - s.add_development_dependency(%q.freeze, [">= 0"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - s.add_development_dependency(%q.freeze, ["~> 3"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - s.add_development_dependency(%q.freeze, ["~> 1"]) - s.add_development_dependency(%q.freeze, ["~> 0"]) - else - s.add_dependency(%q.freeze, [">= 2.3.5"]) - s.add_dependency(%q.freeze, ["~> 3.13.12"]) - s.add_dependency(%q.freeze, ["~> 5.0"]) - s.add_dependency(%q.freeze, [">= 0"]) - s.add_dependency(%q.freeze, ["~> 1.0.3"]) - s.add_dependency(%q.freeze, ["~> 13"]) - s.add_dependency(%q.freeze, ["~> 3.5.0"]) - s.add_dependency(%q.freeze, ["~> 6.1"]) - s.add_dependency(%q.freeze, ["~> 2.1.4"]) - s.add_dependency(%q.freeze, ["~> 2.1.0"]) - s.add_dependency(%q.freeze, [">= 0"]) - s.add_dependency(%q.freeze, ["~> 0"]) - s.add_dependency(%q.freeze, ["~> 3"]) - s.add_dependency(%q.freeze, ["~> 0"]) - s.add_dependency(%q.freeze, ["~> 0"]) - s.add_dependency(%q.freeze, ["~> 1"]) - s.add_dependency(%q.freeze, ["~> 0"]) - end + s.add_runtime_dependency(%q.freeze, [">= 2.3.5"]) + s.add_runtime_dependency(%q.freeze, ["~> 3.16", ">= 3.16.3"]) + s.add_runtime_dependency(%q.freeze, ["~> 5.0"]) + s.add_runtime_dependency(%q.freeze, [">= 0"]) + s.add_runtime_dependency(%q.freeze, ["~> 1.0.3"]) + s.add_runtime_dependency(%q.freeze, ["~> 13"]) + s.add_development_dependency(%q.freeze, ["~> 3.5.0"]) + s.add_development_dependency(%q.freeze, ["~> 6.1"]) + s.add_development_dependency(%q.freeze, ["~> 2.6.7"]) + s.add_development_dependency(%q.freeze, ["~> 2.4.9"]) + s.add_development_dependency(%q.freeze, [">= 0"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) + s.add_development_dependency(%q.freeze, ["~> 3"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) + s.add_development_dependency(%q.freeze, ["~> 1"]) + s.add_development_dependency(%q.freeze, ["~> 0"]) end diff --git a/lib/grade_runner.rb b/lib/grade_runner.rb index c9816bd..b77eab8 100644 --- a/lib/grade_runner.rb +++ b/lib/grade_runner.rb @@ -1,11 +1,19 @@ require "grade_runner/runner" +require "grade_runner/utils/path_utils" +require "grade_runner/services/token_service" +require "grade_runner/services/config_service" +require "grade_runner/services/github_service" +require "grade_runner/services/spec_service" +require "grade_runner/services/grade_service" require "grade_runner/railtie" if defined?(Rails) module GradeRunner class Error < StandardError; end + DEFAULT_SUBMISSION_URL = "https://grades.firstdraft.com" + class << self - attr_writer :default_points, :override_local_specs + attr_writer :default_points, :override_local_specs, :submission_url def default_points @default_points || 1 @@ -19,8 +27,16 @@ def override_local_specs end end + def submission_url + @submission_url || DEFAULT_SUBMISSION_URL + end + def config yield self end + + def project_root + Utils::PathUtils.project_root + end end end diff --git a/lib/grade_runner/formatters/json_output_formatter.rb b/lib/grade_runner/formatters/json_output_formatter.rb index f73fea9..29ec1a7 100644 --- a/lib/grade_runner/formatters/json_output_formatter.rb +++ b/lib/grade_runner/formatters/json_output_formatter.rb @@ -1,5 +1,6 @@ -RSpec::Support.require_rspec_core "formatters/json_formatter" require "oj" +require "rspec/core" + class JsonOutputFormatter < RSpec::Core::Formatters::JsonFormatter RSpec::Core::Formatters.register self, :dump_summary @@ -8,7 +9,7 @@ def dump_summary(summary) examples. map { |example| example.metadata.fetch(:points, GradeRunner.default_points).to_i }. sum - + earned_points = summary. examples. select { |example| example.execution_result.status == :passed }. @@ -17,7 +18,7 @@ def dump_summary(summary) score = (earned_points.to_f / total_points).round(4) score = 0 if score.nan? - + @output_hash[:summary] = { duration: summary.duration, example_count: summary.example_count, @@ -29,14 +30,14 @@ def dump_summary(summary) score: score } result = (@output_hash[:summary][:score] * 100).round(2) - + if summary.errors_outside_of_examples_count.positive? result = "An error occurred while running tests" else result = result.to_s + "%" end - - + + @output_hash[:summary_line] = [ "#{summary.example_count} #{summary.example_count == 1 ? "test" : "tests"}", "#{summary.failure_count} failures", @@ -48,7 +49,7 @@ def dump_summary(summary) def close(_notification) output.write Oj.dump @output_hash end - + private def format_example(example) diff --git a/lib/grade_runner/railtie.rb b/lib/grade_runner/railtie.rb index 1151409..4b2f579 100644 --- a/lib/grade_runner/railtie.rb +++ b/lib/grade_runner/railtie.rb @@ -2,10 +2,6 @@ module GradeRunner class Railtie < ::Rails::Railtie - - rake_tasks do - load "tasks/grade_runner.rake" - load "tasks/grade.rake" - end + rake_tasks { load "grade_runner/tasks.rb" } end end diff --git a/lib/grade_runner/runner.rb b/lib/grade_runner/runner.rb index a21a243..13f300e 100644 --- a/lib/grade_runner/runner.rb +++ b/lib/grade_runner/runner.rb @@ -3,7 +3,6 @@ module GradeRunner class Runner - def initialize(submission_root_url, grades_access_token, rspec_output_json, username, reponame, sha, source) @submission_url = submission_root_url + submission_path @grades_access_token = grades_access_token @@ -47,6 +46,5 @@ def data source: @source } end - end end diff --git a/lib/grade_runner/services/config_service.rb b/lib/grade_runner/services/config_service.rb new file mode 100644 index 0000000..668a891 --- /dev/null +++ b/lib/grade_runner/services/config_service.rb @@ -0,0 +1,55 @@ +require "yaml" +require "fileutils" +require "grade_runner/utils/path_utils" + +module GradeRunner + module Services + class ConfigService + def initialize + @path_utils = GradeRunner::Utils::PathUtils + end + + def find_or_create_directory(directory_name) + @path_utils.find_or_create_directory(directory_name) + end + + def update_config_file(config_file_name, config) + File.write(config_file_name, YAML.dump(config)) + end + + def load_config(config_file_name, default_submission_url) + if File.exist?(config_file_name) + begin + YAML.load_file(config_file_name) + rescue + abort "It looks like there's something wrong with your token in `#{config_file_name}`. Please delete that file and try `rails grade` again, and be sure to provide the access token for THIS project.".red + end + else + { "submission_url" => default_submission_url } + end + end + + def get_config_file_path(directory = ".vscode", filename = ".ltici_apitoken.yml") + config_dir = find_or_create_directory(directory) + "#{config_dir}/#{filename}" + end + + def save_token_to_config(config_file_name, token, submission_url, github_username) + config = { + "submission_url" => submission_url, + "personal_access_token" => token, + "github_username" => github_username + } + update_config_file(config_file_name, config) + end + + def clear_token_in_config(config_file_name) + if File.exist?(config_file_name) + config = YAML.load_file(config_file_name) + config["personal_access_token"] = nil + update_config_file(config_file_name, config) + end + end + end + end +end diff --git a/lib/grade_runner/services/github_service.rb b/lib/grade_runner/services/github_service.rb new file mode 100644 index 0000000..1c27c60 --- /dev/null +++ b/lib/grade_runner/services/github_service.rb @@ -0,0 +1,49 @@ +require "octokit" +require "grade_runner/utils/path_utils" + +module GradeRunner + module Services + class GithubService + def initialize + @path_utils = GradeRunner::Utils::PathUtils + end + + def retrieve_github_username(config_file_name) + if File.exist?(config_file_name) + config = YAML.load_file(config_file_name) + return config["github_username"] if config["github_username"].present? + end + + github_email = `git config user.email`.chomp + return "" if github_email.blank? + + username = `git config user.name`.chomp + search_results = Octokit.search_users("#{github_email} in:email").fetch(:items) + + if search_results.present? + username = search_results.first.fetch(:login, username) + end + + username + end + + def set_upstream_remote(repo_slug) + upstream = `git remote -v | grep -w upstream`.chomp + if upstream.blank? + `git remote add upstream https://github.com/#{repo_slug}` + else + `git remote set-url upstream https://github.com/#{repo_slug}` + end + end + + def get_commit_sha + `git rev-parse HEAD`.slice(0..7) + end + + def get_repo_name + # Extract the repository name from the project path + @path_utils.project_root.to_s.split("/").last + end + end + end +end diff --git a/lib/grade_runner/services/grade_service.rb b/lib/grade_runner/services/grade_service.rb new file mode 100644 index 0000000..30b68f2 --- /dev/null +++ b/lib/grade_runner/services/grade_service.rb @@ -0,0 +1,81 @@ +require "active_support/core_ext/object/blank" + +module GradeRunner + module Services + class GradeService + def initialize + @submission_url = GradeRunner.submission_url + @config_service = ConfigService.new + @token_service = TokenService.new(@submission_url) + @github_service = GithubService.new + @spec_service = SpecService.new + end + + def process_grade_all(input_token = nil) + config_file_name = @config_service.get_config_file_path + config = @config_service.load_config(config_file_name, @submission_url) + + file_token = config["personal_access_token"] + + # Check for Gitpod environment token + if file_token.nil? && ENV.has_key?("LTICI_GITPOD_APITOKEN") + input_token = ENV.fetch("LTICI_GITPOD_APITOKEN") + end + + # Get token (from input, file, or prompt) + token = @token_service.get_token(input_token, file_token, config_file_name) + github_username = @github_service.retrieve_github_username(config_file_name) + + # Save token to config if new + if input_token.present? || (token.present? && file_token.nil?) + @config_service.save_token_to_config(config_file_name, token, @submission_url, github_username) + end + + return "We couldn't find your access token. Please click on the assignment link and run the rails grade command shown there." if token.blank? + + # Validate token + if !@token_service.validate_token(token) + @config_service.clear_token_in_config(config_file_name) + return "Your access token looked invalid, so we've reset it to be blank. Please re-run rails grade and copy-paste your token carefully from the assignment page." + end + + # Sync specs if needed + if GradeRunner.override_local_specs + resource_info = @token_service.fetch_upstream_repo(token) + full_reponame = resource_info.fetch("repo_slug") + remote_spec_folder_sha = resource_info.fetch("spec_folder_sha") + source_code_url = resource_info.fetch("source_code_url") + + @github_service.set_upstream_remote(full_reponame) + @spec_service.sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) + end + + # Run tests + output_path = @spec_service.prepare_output_directory + rspec_output_json = @spec_service.run_tests(output_path) + + # Submit results + username = github_username + reponame = @github_service.get_repo_name + sha = @github_service.get_commit_sha + + GradeRunner::Runner.new(@submission_url, token, rspec_output_json, username, reponame, sha, "manual").process + + "Grade submitted successfully" + end + + def process_reset_token + config_file_name = @config_service.get_config_file_path + github_username = @github_service.retrieve_github_username(config_file_name) + + # Get new token from user + token = @token_service.prompt_for_token(config_file_name) + + # Save token + @config_service.save_token_to_config(config_file_name, token, @submission_url, github_username) + + "Grade token has been reset successfully." + end + end + end +end diff --git a/lib/grade_runner/services/spec_service.rb b/lib/grade_runner/services/spec_service.rb new file mode 100644 index 0000000..7aed4f0 --- /dev/null +++ b/lib/grade_runner/services/spec_service.rb @@ -0,0 +1,140 @@ +require "oj" +require "fileutils" +require "grade_runner/utils/path_utils" +require "active_support/core_ext/object/blank" +require "open-uri" +require "zip" + +module GradeRunner + module Services + class SpecService + def initialize + @path_utils = GradeRunner::Utils::PathUtils + end + + def run_tests(output_path) + # Ensure database is migrated if Rails app + `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) + + # Run tests with JSON formatter and explicitly set path to specs + project_root = Dir.pwd + spec_dir = File.join(project_root, "spec") + args = [ + "--default-path", spec_dir, + "--pattern", "**/*_spec.rb", + "--format", "JsonOutputFormatter", + "--out", output_path, # "tmp/output/results.json" + spec_dir + ] + + # Run RSpec via its Ruby API so you stay in the same process + # This helps with using JsonOutputFormatter + # TODO: handle non-zero exit code + RSpec::Core::Runner.run(args) + + # Load and return test results + Oj.load(File.read(output_path)) + end + + def sync_specs_with_source(full_reponame, remote_sha, repo_url) + # Return if required parameters are missing + return false unless full_reponame && remote_sha && repo_url + + # Unstage staged changes in spec folder + `git restore --staged spec/* 2>/dev/null` + # Discard unstaged changes in spec folder + `git checkout spec -q 2>/dev/null` + `git clean spec -f -q 2>/dev/null` + + # Get local SHA of spec folder + local_sha_output = `git ls-tree HEAD #{@path_utils.project_root}/spec 2>/dev/null`.chomp + local_sha = local_sha_output.split[2] if local_sha_output.present? + + # Only update if remote SHA differs from local SHA + unless remote_sha == local_sha + # Create temporary directories + tmp_dir = @path_utils.find_or_create_directory("tmp") + backup_dir = @path_utils.find_or_create_directory("tmp/backup") + + # Backup existing specs if they exist + if Dir.exist?("#{@path_utils.project_root}/spec") + files_and_subfolders = Dir.glob("#{@path_utils.project_root}/spec/*") + FileUtils.mv(files_and_subfolders, backup_dir) if files_and_subfolders.any? + else + FileUtils.mkdir_p("#{@path_utils.project_root}/spec") + end + + # Download spec zip file + zip_path = "#{tmp_dir}/spec.zip" + download_file(repo_url, zip_path) + + # Extract zip file + extracted_folder = extract_zip(zip_path, tmp_dir) + source_directory = File.join(extracted_folder, "spec") + + # Overwrite spec folder with new specs + overwrite_spec_folder(source_directory) + + # Clean up + FileUtils.rm(zip_path) if File.exist?(zip_path) + FileUtils.rm_rf(extracted_folder) if extracted_folder && Dir.exist?(extracted_folder) + FileUtils.rm_rf(backup_dir) if Dir.exist?(backup_dir) + + # Commit changes + `git add spec/ 2>/dev/null` + `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft " 2>/dev/null` + + return true + end + + false + rescue => e + # In case of error, still try to clean up + tmp_zip = "#{@path_utils.tmp_path}/spec.zip" + FileUtils.rm(tmp_zip) if File.exist?(tmp_zip) + FileUtils.rm_rf("#{@path_utils.tmp_path}/backup") if Dir.exist?("#{@path_utils.tmp_path}/backup") + false + end + + def download_file(url, destination) + download = URI.open(url) + IO.copy_stream(download, destination) + end + + def extract_zip(folder, destination) + extracted_file_path = destination + + Zip::File.open(folder) do |zip_file| + zip_file.each_with_index do |file, index| + # Get name of root folder in zip file + if index == 0 + extracted_file_path = File.join(destination, file.name) + end + + file_path = File.join(destination, file.name) + FileUtils.mkdir_p(File.dirname(file_path)) + file.extract(file_path) unless File.exist?(file_path) + end + end + + extracted_file_path + end + + def overwrite_spec_folder(source_directory) + destination_directory = "#{@path_utils.project_root}/spec" + + # Get all files in the source directory + files = Dir.glob("#{source_directory}/*") + + # Move each file to the destination directory + files.each do |file| + FileUtils.cp_r(file, destination_directory) + end + end + + def prepare_output_directory + @path_utils.tmp_output_path + end + end + end +end diff --git a/lib/grade_runner/services/token_service.rb b/lib/grade_runner/services/token_service.rb new file mode 100644 index 0000000..8a91f20 --- /dev/null +++ b/lib/grade_runner/services/token_service.rb @@ -0,0 +1,74 @@ +require "net/http" +require "oj" +require "active_support/core_ext/object/blank" + +module GradeRunner + module Services + class TokenService + attr_reader :submission_url + + # TODO: do we want to make this configurable? + TOKEN_REGEX = /^[1-9A-Za-z][^OIl]{23}$/ + + def initialize(submission_url) + @submission_url = submission_url + end + + def validate_token(token) + return false unless token.is_a?(String) && token =~ TOKEN_REGEX + + url = "#{submission_url}/submissions/validate_token?token=#{token}" + uri = URI.parse(url) + req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') + res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| + http.request(req) + end + result = Oj.load(res.body) + result["success"] + rescue => e + false + end + + def get_token(input_token, file_token, config_file_name) + if input_token.present? + input_token + elsif file_token.present? + file_token + else + prompt_for_token(config_file_name) + end + end + + def prompt_for_token(config_file_name) + puts "Enter your access token for this project" + new_token = "" + + while new_token.empty? do + print "> " + new_token = $stdin.gets.chomp.strip + + if new_token.empty? || !validate_token(new_token) + puts "Please enter valid token" + new_token = "" + end + end + + new_token + end + + def fetch_upstream_repo(token) + return false unless token.is_a?(String) && token =~ TOKEN_REGEX + + url = "#{submission_url}/submissions/resource?token=#{token}" + uri = URI.parse(url) + req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') + res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| + http.request(req) + end + Oj.load(res.body) + rescue => e + false + end + end + end +end diff --git a/lib/grade_runner/tasks.rb b/lib/grade_runner/tasks.rb new file mode 100644 index 0000000..5cbcd00 --- /dev/null +++ b/lib/grade_runner/tasks.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "rake" unless defined?(Rake) + +# Load any .rake files nested under lib/grade_runner/tasks/ +Dir[File.expand_path("tasks/**/*.rake", __dir__)].sort.each do |file| + load file +end diff --git a/lib/grade_runner/tasks/grade.rake b/lib/grade_runner/tasks/grade.rake new file mode 100644 index 0000000..5a60a1d --- /dev/null +++ b/lib/grade_runner/tasks/grade.rake @@ -0,0 +1,27 @@ +require "active_support/core_ext/object/blank" +require "grade_runner/services/grade_service" + +desc "Alias for \"grade:all\"." +task grade: "grade:all" do +end + +namespace :grade do + desc "Run all tests and submit a build report." + task :all do + ARGV.each { |a| task a.to_sym do ; end } + input_token = ARGV[1] + + grade_service = GradeRunner::Services::GradeService.new + result = grade_service.process_grade_all(input_token) + + puts result if result.present? + end + + desc "Reset access token saved in YAML file." + task :reset_token do + grade_service = GradeRunner::Services::GradeService.new + result = grade_service.process_reset_token + + puts result if result.present? + end +end diff --git a/lib/grade_runner/tasks/grade_runner.rake b/lib/grade_runner/tasks/grade_runner.rake new file mode 100644 index 0000000..7436f12 --- /dev/null +++ b/lib/grade_runner/tasks/grade_runner.rake @@ -0,0 +1,24 @@ +require "grade_runner/services/grade_service" +require "grade_runner/utils/path_utils" +require "grade_runner/formatters/json_output_formatter" +require "grade_runner/formatters/hint_formatter" + +namespace :grade_runner do + desc "Grade project" + task :runner do + # Handle CI-specific paths and environment + rspec_output_json = JSON.parse(File.read("#{ENV['CIRCLE_ARTIFACTS']}/output/rspec_output.json")) + username = ENV["CIRCLE_PROJECT_USERNAME"] + reponame = ENV["CIRCLE_PROJECT_REPONAME"] + sha = ENV["CIRCLE_SHA1"] + token = ENV['GRADES_PERSONAL_ACCESS_TOKEN'] + + if token.present? + # If in CI environment, use Runner directly with CI parameters + submission_url = GradeRunner.submission_url + GradeRunner::Runner.new(submission_url, token, rspec_output_json, username, reponame, sha, 'circle_ci').process + else + puts "We couldn't find your access token, so we couldn't record your grade. Please click on the assignment link again and run the rails grade ... command shown there." + end + end +end diff --git a/lib/grade_runner/test_helpers.rb b/lib/grade_runner/test_helpers.rb new file mode 100644 index 0000000..70db525 --- /dev/null +++ b/lib/grade_runner/test_helpers.rb @@ -0,0 +1,113 @@ +# lib/grade_runner/test_helpers.rb +require "stringio" + +module GradeRunner + ## + # TestHelpers provides small utilities for writing exercise specs + # that need to capture program output. These helpers keep specs + # short, readable, and consistent across projects. + # + # Example usage (RSpec): + # + # # spec/spec_helper.rb + # + # require "bundler/setup" # !important so the Gemfile dependencies are on $LOAD_PATH + # require "grade_runner/test_helpers" + # + # RSpec.configure do |config| + # config.include GradeRunner::TestHelpers + # end + # + # # spec/example_spec.rb + # + # it "captures script output" do + # lines = run_ruby_lines("./hello.rb") + # expect(lines).to eq(["hello, world"]) + # end + # + module TestHelpers + Status = Struct.new(:exitstatus) do + def success? = exitstatus.to_i == 0 + end + + # Run a Ruby script (in-process) and capture stdout, stderr, and status. + # Works with RSpec stubs because no subprocess is spawned. + # + # @param path [String] e.g. "./calculator.rb" + # @param stdin [String] content for gets/$stdin (include trailing "\n"s) + # @param argv [Array] values to replace ARGV with + # @return [Array(String, String, Status)] [stdout, stderr, status] + def run_script(path, stdin: "", argv: []) + orig_stdin, orig_stdout, orig_stderr = $stdin, $stdout, $stderr + orig_argv = ARGV.dup + + in_buf = StringIO.new(String(stdin)) + out_buf = StringIO.new + err_buf = StringIO.new + + $stdin = in_buf + $stdout = out_buf + $stderr = err_buf + ARGV.replace(Array(argv)) + + status = Status.new(0) + + begin + load path + rescue SystemExit => e + status.exitstatus = e.status || 1 + rescue Exception => e + err_buf.puts("#{e.class}: #{e.message}") + e.backtrace&.each { |ln| err_buf.puts(ln) } + status.exitstatus = 1 + ensure + $stdin = orig_stdin + $stdout = orig_stdout + $stderr = orig_stderr + ARGV.replace(orig_argv) + end + + [out_buf.string, err_buf.string, status] + end + alias run_ruby run_script + + # Run a script and return cleaned lines (quotes stripped, trimmed, no blanks). + # + # @return [Array] + def run_script_and_capture_lines(path, stdin: "", argv: []) + stdout, _stderr, _status = run_ruby(path, stdin:, argv:) + clean_output_lines(stdout) + end + alias run_ruby_and_capture_lines run_script_and_capture_lines + + # Run a script and return raw stdout (string). + # + # @return [String] + def capture_raw_stdout_from(path, stdin: "", argv: []) + stdout, _stderr, _status = run_ruby(path, stdin:, argv:) + stdout + end + + # Normalize printed output into human-friendly lines: + # - remove pp quotes + # - split lines + # - strip whitespace + # - drop empties + # + # @param output [String] + # @return [Array] + def normalize_output(output) + output.gsub('"', "").lines.map(&:strip).reject(&:empty?) + end + alias clean_output_lines normalize_output + + # Remove comment-only lines from source text. + # + # @param source [String] + # @return [String] + def strip_comments(source) + source.lines.reject { |line| line.strip.start_with?("#") }.join + end + alias strip_comment_lines strip_comments + end +end diff --git a/lib/grade_runner/utils/path_utils.rb b/lib/grade_runner/utils/path_utils.rb new file mode 100644 index 0000000..45ef49c --- /dev/null +++ b/lib/grade_runner/utils/path_utils.rb @@ -0,0 +1,48 @@ +module GradeRunner + module Utils + class PathUtils + class << self + # Determine the project root directory + # @return [Pathname] The project root path + def project_root + if defined?(Rails) + Rails.root + elsif defined?(Bundler) + Bundler.root + else + Pathname.new(Dir.pwd) + end + end + + # Create a path relative to project root + # @param path [String] Relative path + # @return [Pathname] Absolute path from project root + def path_in_project(path) + project_root.join(path) + end + + # Find or create a directory in the project + # @param directory_name [String] Directory name to find or create + # @return [String] Path to the directory + def find_or_create_directory(directory_name) + directory = path_in_project(directory_name) + FileUtils.mkdir_p(directory) unless Dir.exist?(directory) + directory.to_s + end + + # Get the temporary output directory path for test results + # @return [String] Path to the output directory + def tmp_output_path + output_dir = find_or_create_directory("tmp/output") + File.join(output_dir, "#{Time.now.to_i}.json") + end + + # Get the temporary directory path + # @return [String] Path to the temporary directory + def tmp_path + find_or_create_directory("tmp") + end + end + end + end +end diff --git a/lib/tasks/grade.rake b/lib/tasks/grade.rake deleted file mode 100644 index 34114a8..0000000 --- a/lib/tasks/grade.rake +++ /dev/null @@ -1,276 +0,0 @@ -require "active_support/core_ext/object/blank" -require "grade_runner/runner" -require "octokit" -require "yaml" -require "zip" -require "fileutils" -require "open-uri" - -desc "Alias for \"grade:next\"." -task grade: "grade:all" do -end - -namespace :grade do - desc "Run all tests and submit a build report." - task :all do - ARGV.each { |a| task a.to_sym do ; end } - input_token = ARGV[1] - file_token = nil - - config_dir_name = find_or_create_directory(".vscode") - config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - student_config = {} - student_config["submission_url"] = "https://grades.firstdraft.com" - student_config["github_username"] = retrieve_github_username - - if File.exist?(config_file_name) - begin - config = YAML.load_file(config_file_name) - rescue - abort "It looks like there's something wrong with your token in `#{config_dir_name}/.ltici_apitoken.yml`. Please delete that file and try `rails grade` again, and be sure to provide the access token for THIS project.".red - end - submission_url = config["submission_url"] - file_token = config["personal_access_token"] - student_config["submission_url"] = config["submission_url"] - else - submission_url = "https://grades.firstdraft.com" - end - if file_token.nil? && ENV.has_key?("LTICI_GITPOD_APITOKEN") - input_token = ENV.fetch("LTICI_GITPOD_APITOKEN") - end - if input_token.present? - token = input_token - student_config["personal_access_token"] = input_token - update_config_file(config_file_name, student_config) - elsif input_token.nil? && file_token.present? - token = file_token - elsif input_token.nil? && file_token.nil? - puts "Enter your access token for this project" - new_personal_access_token = "" - - while new_personal_access_token == "" do - print "> " - new_personal_access_token = $stdin.gets.chomp.strip - if new_personal_access_token!= "" && is_valid_token?(submission_url, new_personal_access_token) == false - puts "Please enter valid token" - new_personal_access_token = "" - end - - if new_personal_access_token != "" - student_config["personal_access_token"] = new_personal_access_token - update_config_file(config_file_name, student_config) - token = new_personal_access_token - end - end - end - - if token.present? - - if is_valid_token?(submission_url, token) == false - student_config["personal_access_token"] = nil - update_config_file(config_file_name, student_config) - puts "Your access token looked invalid, so we've reset it to be blank. Please re-run rails grade and, when asked, copy-paste your token carefully from the assignment page." - else - if GradeRunner.override_local_specs - resource_info = upstream_repo(submission_url, token) - full_reponame = resource_info.fetch("repo_slug") - remote_spec_folder_sha = resource_info.fetch("spec_folder_sha") - source_code_url = resource_info.fetch("source_code_url") - set_upstream_remote(full_reponame) - sync_specs_with_source(full_reponame, remote_spec_folder_sha, source_code_url) - end - - path = File.join(project_root, "/tmp/output/#{Time.now.to_i}.json") - `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - `RAILS_ENV=test bundle exec rspec --format JsonOutputFormatter --out #{path}` - rspec_output_json = Oj.load(File.read(path)) - username = retrieve_github_username - reponame = project_root.to_s.split("/").last - sha = `git rev-parse HEAD`.slice(0..7) - - GradeRunner::Runner.new(submission_url, token, rspec_output_json, username, reponame, sha, "manual").process - end - else - puts "We couldn't find your access token, so we couldn't record your grade. Please click on the assignment link again and run the rails grade ... command shown there." - end - end - - desc "Run only the next failing test." - task :next do - path = File.join(project_root, "examples.txt") - if File.exist?(path) - `bin/rails db:migrate RAILS_ENV=test` if defined?(Rails) - puts `RAILS_ENV=test bundle exec rspec --next-failure --format HintFormatter` - else - puts `RAILS_ENV=test bundle exec rspec` - puts "Please rerun rails grade:next to run the first failing spec" - end - end - - desc "Reset access token saved in YAML file." - task :reset_token do - config_dir_name = find_or_create_directory(".vscode") - config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - submission_url = "https://grades.firstdraft.com" - - student_config = {} - student_config["submission_url"] = submission_url - student_config["github_username"] = retrieve_github_username - puts "Enter your access token for this project" - new_personal_access_token = "" - - while new_personal_access_token == "" do - print "> " - new_personal_access_token = $stdin.gets.chomp.strip - token_valid = is_valid_token?(submission_url, new_personal_access_token) - unless token_valid && !new_personal_access_token.empty? - puts "Please enter valid token" - new_personal_access_token = "" - end - - unless new_personal_access_token.empty? - student_config["personal_access_token"] = new_personal_access_token - update_config_file(config_file_name, student_config) - token = new_personal_access_token - end - end - puts "Grade token has been reset successfully." - end - -end - -def sync_specs_with_source(full_reponame, remote_sha, repo_url) - # Unstage staged changes in spec folder - `git restore --staged spec/* ` - # Discard unstaged changes in spec folder - `git checkout spec -q` - `git clean spec -f -q` - local_sha = `git ls-tree HEAD #{project_root.join('spec')}`.chomp.split[2] - - unless remote_sha == local_sha - find_or_create_directory("tmp") - find_or_create_directory("tmp/backup") - files_and_subfolders_inside_specs = Dir.glob("spec/*") - # Temporarily move specs - FileUtils.mv(files_and_subfolders_inside_specs, "tmp/backup") - - download_file(repo_url, "tmp/spec.zip") - extracted_zip_folder = extract_zip("tmp/spec.zip", "tmp") - source_directory = extracted_zip_folder.join("spec") - overwrite_spec_folder(source_directory) - - FileUtils.rm(project_root.join("tmp/spec.zip")) - FileUtils.rm_rf(extracted_zip_folder) - FileUtils.rm_rf("tmp/backup") - `git add spec/` - `git commit spec/ -m "Update spec/ folder to latest version" --author "First Draft "` - end -end - -def download_file(url, destination) - download = URI.open(url) - IO.copy_stream(download, destination) -end - -def extract_zip(folder, destination) - extracted_file_path = project_root.join(destination) - Zip::File.open(folder) do |zip_file| - zip_file.each_with_index do |file, index| - # Get name of root folder in zip file - if index == 0 - extracted_file_path = extracted_file_path.join(file.name) - end - file_path = File.join(destination, file.name) - FileUtils.mkdir_p(File.dirname(file_path)) - file.extract(file_path) - end - end - extracted_file_path -end - -def overwrite_spec_folder(source_directory) - destination_directory = "spec" - # Get all files in the source directory - files = Dir.glob("#{source_directory}/*") - # Move each file to the destination directory - files.each do |file| - FileUtils.mv(file, destination_directory) - end -end - -def set_upstream_remote(repo_slug) - upstream = `git remote -v | grep -w upstream`.chomp - if upstream.blank? - `git remote add upstream https://github.com/#{repo_slug}` - else - `git remote set-url upstream https://github.com/#{repo_slug}` - end -end - -def update_config_file(config_file_name, config) - File.write(config_file_name, YAML.dump(config)) -end - -def find_or_create_directory(directory_name) - directory = File.join(project_root, directory_name) - Dir.mkdir(directory) unless Dir.exist?(directory) - directory -end - -def is_valid_token?(root_url, token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ - url = "#{root_url}/submissions/validate_token?token=#{token}" - uri = URI.parse(url) - req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') - res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| - http.request(req) - end - result = Oj.load(res.body) - result["success"] -rescue => e - return false -end - -def upstream_repo(root_url, token) - return false unless token.is_a?(String) && token =~ /^[1-9A-Za-z][^OIl]{23}$/ - url = "#{root_url}/submissions/resource?token=#{token}" - uri = URI.parse(url) - req = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json') - res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| - http.request(req) - end - Oj.load(res.body) -rescue => e - return false -end - -def retrieve_github_username - config_dir_name = find_or_create_directory(".vscode") - config_file_name = "#{config_dir_name}/.ltici_apitoken.yml" - if File.exist?(config_file_name) - config = YAML.load_file(config_file_name) - if config["github_username"].present? - return config["github_username"] - end - else - github_email = `git config user.email`.chomp - return "" if github_email.blank? - username = `git config user.name`.chomp - search_results = Octokit.search_users("#{github_email} in:email").fetch(:items) - if search_results.present? - username = search_results.first.fetch(:login, username) - end - return username - end -end - -def project_root - if defined?(Rails) - return Rails.root - end - - if defined?(Bundler) - return Bundler.root - end - Dir.pwd -end diff --git a/lib/tasks/grade_runner.rake b/lib/tasks/grade_runner.rake deleted file mode 100644 index ddde8a2..0000000 --- a/lib/tasks/grade_runner.rake +++ /dev/null @@ -1,25 +0,0 @@ -namespace :grade_runner do - desc "Grade project" - task :runner do - default_submission_url = "https://grades.firstdraft.com" - config = {} - path = Rails.root.join("grades.yml") - if File.exist?(path) - begin - config = YAML.load_file(path) - rescue - abort "It looks like there's something wrong with your token in `/grades.yml`. Please delete that file and try `rails grade:all` again, and be sure to provide the access token for THIS project.".red - end - end - rspec_output_json = JSON.parse(File.read("#{ENV['CIRCLE_ARTIFACTS']}/output/rspec_output.json")) - username = ENV["CIRCLE_PROJECT_USERNAME"] - reponame = ENV["CIRCLE_PROJECT_REPONAME"] - sha = ENV["CIRCLE_SHA1"] - token = ENV['GRADES_PERSONAL_ACCESS_TOKEN'] - if token.present? - GradeRunner::Runner.new('', config['submission_url'] || default_submission_url, token, rspec_output_json, username, reponame, sha, 'circle_ci').process - else - puts "We couldn't find your access token, so we couldn't record your grade. Please click on the assignment link again and run the rails grade ... command shown there." - end - end -end