<%= f.label :owner_id %>
<% if policy(@evaluator).transfer? %>
diff --git a/app/views/evaluators/show.html.erb b/app/views/evaluators/show.html.erb
index d91657de..3835fda5 100644
--- a/app/views/evaluators/show.html.erb
+++ b/app/views/evaluators/show.html.erb
@@ -10,6 +10,8 @@
Language:
<%= @evaluator.language&.name %>
+ Interactive processes:
+ <%= @evaluator.interactive_processes %>
Source:
<%= predisplay @evaluator.source, language: @evaluator.language&.lexer %>
diff --git a/app/workers/judge_submission_worker.rb b/app/workers/judge_submission_worker.rb
index 8d862905..a3d11317 100644
--- a/app/workers/judge_submission_worker.rb
+++ b/app/workers/judge_submission_worker.rb
@@ -134,7 +134,7 @@ def judge
end
private
- attr_accessor :problem, :box, :tmpdir
+ attr_accessor :problem, :box, :interactive_boxes, :tmpdir
def time_limit
problem.time_limit || 0.001
@@ -154,18 +154,22 @@ def extra_time
end
def wall_time
- time_limit*3+extra_time+5
+ time_limit*2+extra_time+1
end
def setup_judging
self.problem = submission.problem
+ self.box = Isolate.new
+ self.interactive_boxes = Array.new(problem.evaluator&.interactive_processes || 0) { Isolate.new }
Dir.mktmpdir do |tmpdir|
self.tmpdir = tmpdir
- Isolate.box do |box|
- self.box = box
- yield
- end or raise Isolate::LockError, "Error locking box"
+ yield
end
+ ensure
+ box.send(:destroy) if !box.nil?
+ interactive_boxes.each do |box|
+ box.send(:destroy) if !box.nil?
+ end if !interactive_boxes.nil?
end
def compile!(source, language, output)
@@ -177,15 +181,19 @@ def compile!(source, language, output)
end
def judge_test_case(test_case, run_command, eval_command, resource_limits)
- FileUtils.copy(File.expand_path(exe_filename, tmpdir), box.expand_path(exe_filename))
- result = run_test_case(test_case, run_command, resource_limits)
- result['evaluator'] = evaluate_output(test_case, result['output'], result['output_size'], eval_command)
+ if problem.evaluator&.interactive_processes&.positive?
+ result = run_interactive(test_case, run_command, eval_command, resource_limits)
+ else
+ result = run_test_case(test_case, run_command, resource_limits)
+ result['evaluator'] = evaluate_output(test_case, result['output'], result['output_size'], eval_command)
+ end
result['log'] = truncate_output(result['log']) # log only a small portion
result['output'] = truncate_output(result['output'].slice(0,100)) # store only a small portion
result
end
def run_test_case(test_case, run_command, resource_limits = {})
+ FileUtils.copy(File.expand_path(exe_filename, tmpdir), box.expand_path(exe_filename))
stream_limit = OutputBaseLimit + test_case.output.bytesize*2
run_opts = resource_limits.reverse_merge(:output_limit => stream_limit, :clean_utf8 => true)
if submission.input.nil?
@@ -254,6 +262,144 @@ def evaluate_output(test_case, output, output_size, eval_command)
box.clean!
end
+ def run_interactive(test_case, run_command, eval_command, resource_limits)
+ stream_limit = OutputBaseLimit + test_case.output.bytesize*2
+ num_processes = problem.evaluator.interactive_processes
+
+ metafiles = []
+ boxfiles = []
+ logfiles = []
+ manager_pipes = []
+ user_pids = []
+
+ interactive_boxes.each_with_index do |box, index|
+ FileUtils.copy(File.expand_path(exe_filename, tmpdir), box.expand_path(exe_filename))
+ metafiles << Tempfile.new('metafile')
+ boxfiles << Tempfile.new('boxfile')
+ logfiles << box.tmpfile
+ manager_to_user = IO.pipe
+ user_to_manager = IO.pipe
+ manager_pipes << [user_to_manager[0], manager_to_user[1]]
+
+ run_opts = resource_limits.merge(:meta => metafiles.last.path, :err => boxfiles.last, :in => manager_to_user[0], :out => user_to_manager[1], :stderr => logfiles.last)
+ box_run_command = run_command
+ if num_processes > 1
+ box_run_command += " " + index.to_s
+ end
+ user_pids << box.spawn_command(box_run_command, run_opts)
+
+ manager_to_user[0].close
+ user_to_manager[1].close
+ end
+
+ expected = conditioned_output(test_case.output)
+ FileUtils.copy(File.expand_path(EvalFileName, tmpdir), box.expand_path(EvalFileName))
+ manager_resource_limits = { :mem => 524288, :time => num_processes * time_limit + 15, :wall_time => num_processes * wall_time + 30 }
+
+ eval_output = nil
+ eval_result = {}
+ str_to_pipe(expected) do |output_stream|
+ run_opts = manager_resource_limits.merge(:processes => true, 4 => output_stream, :stdin_data => test_case.input, :output_limit => OutputBaseLimit + test_case.output.bytesize*4, :clean_utf8 => true, :inherit_fds => true)
+ manager_pipes.each_with_index do |pipes, index|
+ run_opts.merge!(index * 2 + 5 => pipes[0], index * 2 + 6 => pipes[1])
+ end
+ (stdout,), (eval_result['log'],eval_result['log_size']), (eval_result['box'],), eval_result['meta'], status = box.capture5("#{eval_command}", run_opts)
+ eval_result['log'] = truncate_output(eval_result['log'])
+ eval_output = stdout.strip.split(nil,2)
+ eval_result['stat'] = status.exitstatus
+ end
+
+ manager_pipes.each do |pipes|
+ pipes[0].close
+ pipes[1].close
+ end
+
+ if eval_output.empty?
+ eval_result.delete('evaluation') # error
+ else
+ eval_result['evaluation'] = eval_output[0].to_d
+ eval_result['message'] = truncate_output(eval_output[1] || "")
+ end
+
+ r = { 'evaluator' => eval_result }
+
+ r['stat'] = 0
+ user_pids.each do |pid|
+ exit_status = Process.detach(pid).value.exitstatus
+ r['stat'] = r['stat'].nonzero? || exit_status
+ end
+
+ metafiles.each do |metafile|
+ metafile.open
+ meta = Isolate.parse_meta(metafile.read)
+
+ if r['meta'].nil?
+ r['meta'] = meta
+ else
+ # Merge execution stats
+ r['meta']['time'] += meta['time']
+ r['meta']['time-wall'] = [r['meta']['time-wall'], meta['time-wall']].max
+ r['meta']['cg-mem'] += meta['cg-mem'] if meta.has_key?('cg-mem')
+ r['meta']['max-rss'] += meta['max-rss']
+
+ # Use first non-OK status and related values
+ if r['meta']['status'] == 'OK' && meta['status'] != 'OK'
+ r['meta']['status'] = meta['status']
+ r['meta']['killed'] = meta['killed'] if meta.has_key?('killed')
+ r['meta']['exitcode'] = meta['exitcode'] if meta.has_key?('exitcode')
+ r['meta']['exitsig'] = meta['exitsig'] if meta.has_key?('exitsig')
+ r['meta']['message'] = meta['message'] if meta.has_key?('message')
+ end
+ end
+ metafile.close
+ end
+
+ # Check for MLE/TLE based on merged execution stats
+ if r['meta']['status'] == 'OK'
+ if (r['meta']['cg-mem'] || r['meta']['max-rss']).to_f > memory_limit*1024
+ r['meta']['status'] = 'SG'
+ r['meta']['exitsig'] = 9
+ r['meta']['message'] = "Memory Limit Exceeded"
+ r['meta']['cg-mem'] = [r['meta']['cg-mem'],memory_limit*1024].min if r['meta'].has_key?('cg-mem')
+ r['meta']['max-rss'] = [r['meta']['max-rss'],memory_limit*1024].min
+ r['stat'] = 1
+ elsif r['meta']['time'] > time_limit.to_f
+ r['meta']['status'] = 'TO'
+ r['meta']['message'] = "Time Limit Exceeded"
+ r['stat'] = 1
+ end
+ end
+
+ r['box'] = ''
+ boxfiles.each do |boxfile|
+ boxfile.open
+ r['box'] << boxfile.read
+ boxfile.close
+ end
+
+ r['log'] = ''
+ r['log_size'] = 0
+ interactive_boxes.zip(logfiles).each do |box, logfile|
+ stderr = File.open(box.expand_path(logfile)) { |f| box.read_pipe_limited(f, stream_limit) }
+ (log, log_size) = box.clean_utf8(stderr)
+ r['log'] << log
+ r['log_size'] += log_size
+ end
+ r['log'] = truncate_output(r['log'])
+
+ r['output'] = '' # no user output
+ r['output_size'] = 0
+
+ r['time'] = [r['meta']['time'],time_limit.to_f].min
+
+ return r
+ ensure
+ metafiles.each(&:close!)
+ boxfiles.each(&:close!)
+ interactive_boxes.each(&:clean!)
+ box.clean!
+ end
+
def grade_test_set test_set, evaluated_test_cases
result = {'cases' => []}
pending, error, sig = false, false, false
diff --git a/db/migrate/20230225064438_add_interactive_processes_to_evaluators.rb b/db/migrate/20230225064438_add_interactive_processes_to_evaluators.rb
new file mode 100644
index 00000000..fdaa86fc
--- /dev/null
+++ b/db/migrate/20230225064438_add_interactive_processes_to_evaluators.rb
@@ -0,0 +1,5 @@
+class AddInteractiveProcessesToEvaluators < ActiveRecord::Migration
+ def change
+ add_column :evaluators, :interactive_processes, :integer, null: false, default: 0
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0c980263..cb0af18d 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20230225054132) do
+ActiveRecord::Schema.define(version: 20230225064438) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -87,13 +87,14 @@
end
create_table "evaluators", force: :cascade do |t|
- t.string "name", null: false
- t.text "description", default: "", null: false
- t.text "source", default: "", null: false
- t.integer "owner_id", null: false
+ t.string "name", null: false
+ t.text "description", default: "", null: false
+ t.text "source", default: "", null: false
+ t.integer "owner_id", null: false
t.datetime "created_at"
t.datetime "updated_at"
t.integer "language_id"
+ t.integer "interactive_processes", default: 0, null: false
end
create_table "file_attachments", force: :cascade do |t|
diff --git a/spec/factories/problems.rb b/spec/factories/problems.rb
index d50edc1e..13a10226 100644
--- a/spec/factories/problems.rb
+++ b/spec/factories/problems.rb
@@ -4,34 +4,173 @@
factory :problem do
sequence(:name) {|n| "Problem #{n}" }
statement { "Do nothing" }
- sequence(:input) {|n| "#{n}.in" }
- sequence(:output) {|n| "#{n}.out"}
+ input { nil }
+ output { nil }
memory_limit { 1 }
time_limit { 1 }
owner_id { 0 }
- factory :adding_problem do
+
+ test_sets { test_cases.map{FactoryBot.create(:test_set)} }
+
+ after(:create) do |problem|
+ problem.test_cases.zip(problem.test_sets).each do |test_case, test_set|
+ FactoryBot.create(:test_case_relation, :test_case => test_case, :test_set => test_set)
+ end
+ end
+
+ factory :adding_problem_stdio do
sequence(:name) {|n| "Adding problem #{n}" }
statement { "Read two integers from input and output the sum." }
- input { "add.in" }
- output { "add.out" }
memory_limit { 30 }
time_limit { 1 }
test_cases { [FactoryBot.create(:test_case, :input => "5 9", :output => "14"),
FactoryBot.create(:test_case, :input => "100 -50", :output => "50"),
FactoryBot.create(:test_case, :input => "1235 942", :output => "2177"),
FactoryBot.create(:test_case, :input => "-4000 123", :output => "-3877")] }
- test_sets { (0...4).map{FactoryBot.create(:test_set)} }
- after(:create) do |problem|
- (0...4).each do |i|
- FactoryBot.create(:test_case_relation, :test_case => problem.test_cases[i], :test_set => problem.test_sets[i])
- end
+ factory :adding_problem do
+ input { "add.in" }
+ output { "add.out" }
end
+ end
- factory :adding_problem_stdio do
- input { nil }
- output { nil }
- end
+ factory :binary_search_problem do
+ name { "Binary search problem" }
+ statement { "Find the target number within Q guesses. After each guess you are told whether the target is lower, higher, or correct." }
+ memory_limit { 16 }
+ time_limit { 0.1 }
+ test_cases { [FactoryBot.create(:test_case, :input => "100 100 98"),
+ FactoryBot.create(:test_case, :input => "100000 17 37")] }
+
+ evaluator { FactoryBot.create(:evaluator, :language => LanguageGroup.find_by_identifier("c++").current_language,
+ :interactive_processes => 1, :source => <<~'sourcecode' ) }
+ #include
+ #include
+ #include
+
+ void grade(int score, const char* message = NULL) {
+ fprintf(stdout, "%d\n", score);
+ if (message)
+ fprintf(stderr, "%s\n", message);
+ exit(0);
+ }
+
+ int main() {
+ {
+ // Keep alive on broken pipes
+ struct sigaction sa;
+ sa.sa_handler = SIG_IGN;
+ sigaction(SIGPIPE, &sa, NULL);
+ }
+
+ FILE* user_in = fdopen(5, "r");
+ FILE* user_out = fdopen(6, "w");
+
+ int N, Q, K;
+ scanf("%d %d %d", &N, &Q, &K);
+ fclose(stdin);
+ fprintf(user_out, "%d %d\n", N, Q);
+ fflush(user_out);
+
+ int guess;
+ for (int i = 0; i < Q; ++i) {
+ if (fscanf(user_in, "%d", &guess) != 1) {
+ grade(0, "Could not read guess");
+ }
+ if (guess == K) {
+ fprintf(user_out, "0\n");
+ fflush(user_out);
+ break;
+ } else if (guess < K) {
+ fprintf(user_out, "1\n");
+ fflush(user_out);
+ } else {
+ fprintf(user_out, "-1\n");
+ fflush(user_out);
+ }
+ if (i == Q - 1) {
+ grade(0, "Too many guesses");
+ }
+ }
+
+ if (fscanf(user_in, "%d", &guess) != EOF)
+ grade(0, "Wrong output format, trailing garbage");
+
+ grade(1);
+ }
+ sourcecode
+ end
+
+ factory :integer_encoding_problem do
+ name { "Integer encoding problem" }
+ statement { "Send the input number between two processes using only alphabetic characters." }
+ memory_limit { 16 }
+ time_limit { 0.5 }
+ test_cases { [FactoryBot.create(:test_case, :input => "0"),
+ FactoryBot.create(:test_case, :input => "42"),
+ FactoryBot.create(:test_case, :input => "9999")] }
+
+ evaluator { FactoryBot.create(:evaluator, :interactive_processes => 2, :source => <<~'sourcecode' ) }
+ #!/usr/bin/env python3
+ import os
+ import sys
+ import functools
+ import traceback
+ import time
+
+ print = functools.partial(print, flush=True) # Always flush
+
+ user1_in = os.fdopen(5, 'r')
+ user1_out = os.fdopen(6, 'w')
+ user2_in = os.fdopen(7, 'r')
+ user2_out = os.fdopen(8, 'w')
+
+ def grade(score, admin_message=None, user_message=None):
+ if not user2_out.closed:
+ try:
+ print(-1, file=user2_out)
+ except:
+ pass
+ print(score)
+ if user_message is not None:
+ print(user_message)
+ if admin_message is not None:
+ print(admin_message, file=sys.stderr)
+ sys.exit(0)
+
+ N = int(input())
+
+ try:
+ print(1, file=user1_out)
+ print(N, file=user1_out)
+ user1_out.close()
+ encoded_string = user1_in.read(100000).strip()
+ except (BrokenPipeError, ValueError):
+ grade(0, traceback.format_exc())
+
+ if user1_in.read(100000).strip():
+ grade(0, "Wrong output format, trailing garbage")
+ user1_in.close()
+
+ if not encoded_string.isalpha():
+ grade(0, "Invalid encoded string: " + encoded_string)
+
+ try:
+ print(2, file=user2_out)
+ print(encoded_string, file=user2_out)
+ user2_out.close()
+ decoded_integer = int(user2_in.readline(100000))
+ except (BrokenPipeError, ValueError):
+ grade(0, traceback.format_exc())
+
+ if user2_in.read(100000).strip():
+ grade(0, "Wrong output format, trailing garbage")
+ user2_in.close()
+
+ if decoded_integer != N:
+ grade(0, "Wrong answer")
+ grade(1)
+ sourcecode
end
end
end
diff --git a/spec/factories/submissions.rb b/spec/factories/submissions.rb
index f658bbc6..33c9c869 100644
--- a/spec/factories/submissions.rb
+++ b/spec/factories/submissions.rb
@@ -70,6 +70,128 @@
fprintf(out, "%u\\n",(int)c);
return 0;
}
+sourcecode
+ end
+ factory :binary_search_submission do
+ language { LanguageGroup.find_by_identifier("c++").current_language }
+ source { <
+ using namespace std;
+ int main() {
+ int lo=0, hi, attempts, result;
+ cin >> hi >> attempts;
+ while (hi - lo > 1) {
+ int mid = (lo + hi) / 2;
+ cout << mid << endl;
+ cin >> result;
+ if ( result == 0 )
+ return 0;
+ else if ( result < 0 )
+ hi = mid;
+ else
+ lo = mid + 1;
+ }
+ cout << lo << endl;
+ }
+sourcecode
+ end
+ factory :binary_search_submission_incorrect do
+ language { LanguageGroup.find_by_identifier("c++").current_language }
+ source { <
+ using namespace std;
+ int main() {
+ int hi, attempts, result;
+ cin >> hi >> attempts;
+ for (int i = 0; i < hi; i++) {
+ cout << i << endl;
+ cin >> result;
+ if (result == 0)
+ break;
+ }
+ }
+sourcecode
+ end
+ factory :binary_search_submission_wall_tle do
+ language { LanguageGroup.find_by_identifier("c++").current_language }
+ source { <
+ using namespace std;
+ int main() {
+ int hi, attempts, result;
+ cin >> hi >> attempts;
+ for (int i = 0; i < hi; i++) {
+ //cout << i << endl;
+ cin >> result;
+ }
+ }
+sourcecode
+ end
+ factory :integer_encoding_submission do
+ language { LanguageGroup.find_by_identifier("c++").current_language }
+ source { <
+ #include
+ #include
+ using namespace std;
+ int main() {
+ int mode, N = 0;
+ std::string encoded_string;
+ cin >> mode;
+ if (mode == 1) {
+ cin >> N;
+ while (N) {
+ encoded_string += 'a' + (N & 1);
+ N >>= 1;
+ }
+ encoded_string += 'a';
+ reverse(encoded_string.begin(), encoded_string.end());
+ cout << encoded_string << endl;
+ }
+ if (mode == 2) {
+ cin >> encoded_string;
+ for (char c : encoded_string) {
+ N <<= 1;
+ N += c > 'a';
+ }
+ cout << N << endl;
+ }
+ }
+sourcecode
+ end
+ factory :integer_encoding_submission_mle do
+ language { LanguageGroup.find_by_identifier("c++").current_language }
+ source { <
+ #include
+ #include
+ using namespace std;
+ int main() {
+ std::array arr; // x2 = 20 MiB; should MLE
+ arr.fill(-1);
+
+ int mode, N = 0;
+ std::string encoded_string;
+ cin >> mode;
+ if (mode == 1) {
+ cin >> N;
+ while (N) {
+ encoded_string += 'a' + (N & 1);
+ N >>= 1;
+ }
+ encoded_string += 'a';
+ reverse(encoded_string.begin(), encoded_string.end());
+ cout << encoded_string << endl;
+ }
+ if (mode == 2) {
+ cin >> encoded_string;
+ for (char c : encoded_string) {
+ N <<= 1;
+ N += c > 'a';
+ }
+ cout << N << endl;
+ }
+ }
sourcecode
end
end
diff --git a/spec/models/submission_spec.rb b/spec/models/submission_spec.rb
index dd66ff1d..ab8c0bff 100644
--- a/spec/models/submission_spec.rb
+++ b/spec/models/submission_spec.rb
@@ -40,4 +40,60 @@
expect(@unsigned_submission.evaluation).to eq(0.75)
end
end
+
+ context 'on "binary search" problem' do
+ before(:all) do
+ @user = FactoryBot.create(:user)
+ @problem = FactoryBot.create(:binary_search_problem)
+ end
+ after(:all) do
+ [@user, @problem].reverse_each { |object| object.destroy }
+ end
+ it 'judges submission' do
+ submission = FactoryBot.create(:binary_search_submission, :problem => @problem, :user => @user)
+ expect(submission.score).to be_nil
+ submission.judge
+ submission.reload
+ expect(submission.evaluation).to eq(1)
+ end
+ it 'judges incorrect submission' do
+ submission = FactoryBot.create(:binary_search_submission_incorrect, :problem => @problem, :user => @user)
+ expect(submission.evaluation).to be_nil
+ submission.judge
+ submission.reload
+ expect(submission.evaluation).to eq(0.5)
+ end
+ it 'judges wall timeout submission' do
+ submission = FactoryBot.create(:binary_search_submission_wall_tle, :problem => @problem, :user => @user)
+ expect(submission.evaluation).to be_nil
+ submission.judge
+ submission.reload
+ expect(submission.evaluation).to eq(0)
+ end
+ end
+
+ context 'on "integer encoding" problem' do
+ before(:all) do
+ @user = FactoryBot.create(:user)
+ @problem = FactoryBot.create(:integer_encoding_problem)
+ end
+ after(:all) do
+ [@user, @problem].reverse_each { |object| object.destroy }
+ end
+ it 'judges submission' do
+ submission = FactoryBot.create(:integer_encoding_submission, :problem => @problem, :user => @user)
+ expect(submission.score).to be_nil
+ submission.judge
+ submission.reload
+ expect(submission.evaluation).to eq(1)
+ end
+ it 'judges memory limit exceeded submission' do
+ submission = FactoryBot.create(:integer_encoding_submission_mle, :problem => @problem, :user => @user)
+ expect(submission.score).to be_nil
+ submission.judge
+ submission.reload
+ expect(submission.judge_data.test_cases.first[1].status).to eq(:memory)
+ expect(submission.evaluation).to eq(0)
+ end
+ end
end