docker cleaner
This commit is contained in:
Egor Matveev
parent
bd8eef9325
commit
93640aa1ae
@ -39,23 +39,32 @@ class BaseTester:
|
||||
)
|
||||
if code != 0:
|
||||
raise TestException("RE")
|
||||
result = open(join(self.path, "output.txt"), "r").read().strip().replace('\r\n', '\n')
|
||||
result = (
|
||||
open(join(self.path, "output.txt"), "r")
|
||||
.read()
|
||||
.strip()
|
||||
.replace("\r\n", "\n")
|
||||
)
|
||||
print("got result", result)
|
||||
if self.checker_code is not None:
|
||||
print('using checker')
|
||||
with open(join(self.path, 'expected.txt'), 'w') as fs:
|
||||
print("using checker")
|
||||
with open(join(self.path, "expected.txt"), "w") as fs:
|
||||
fs.write(self.predicted)
|
||||
with open(join(self.path, 'checker.py'), 'w') as fs:
|
||||
with open(join(self.path, "checker.py"), "w") as fs:
|
||||
fs.write(self.checker_code)
|
||||
code = call(f'docker exec -i solution_{self.solution.id}_checker sh -c "cd app && python checker.py"', shell=True, timeout=1)
|
||||
code = call(
|
||||
f'docker exec -i solution_{self.solution.id}_checker sh -c "cd app && python checker.py"',
|
||||
shell=True,
|
||||
timeout=1,
|
||||
)
|
||||
if code != 0:
|
||||
raise TestException("WA")
|
||||
else:
|
||||
print('using simple check')
|
||||
print("using simple check")
|
||||
if result != self.predicted:
|
||||
print('incorrect')
|
||||
print("incorrect")
|
||||
raise TestException("WA")
|
||||
print('correct')
|
||||
print("correct")
|
||||
|
||||
def after_test(self):
|
||||
pass
|
||||
@ -71,7 +80,7 @@ class BaseTester:
|
||||
def call(self, command):
|
||||
print(f"Executing command: {command}")
|
||||
if exists(self.path):
|
||||
return call(f'cd {self.path} && {command}', shell=True)
|
||||
return call(f"cd {self.path} && {command}", shell=True)
|
||||
else:
|
||||
return call(command, shell=True)
|
||||
|
||||
@ -85,15 +94,17 @@ class BaseTester:
|
||||
self.call(f"docker network create solution_network_{self.solution.id}")
|
||||
for file in self.solution.task.dockerfiles:
|
||||
add_name = file.filename[11:]
|
||||
with open(join(self.path, 'Dockerfile'), 'w') as fs:
|
||||
with open(join(self.path, "Dockerfile"), "w") as fs:
|
||||
fs.write(file.text)
|
||||
self.call(f"docker build -t solution_image_{self.solution.id}_{add_name} .")
|
||||
run_command = f"docker run "\
|
||||
f"--hostname {add_name} "\
|
||||
f"--network solution_network_{self.solution.id} "\
|
||||
f"--name solution_container_{self.solution.id}_{add_name} "\
|
||||
f"-t -d solution_image_{self.solution.id}_{add_name}"
|
||||
print('run command', run_command)
|
||||
run_command = (
|
||||
f"docker run "
|
||||
f"--hostname {add_name} "
|
||||
f"--network solution_network_{self.solution.id} "
|
||||
f"--name solution_container_{self.solution.id}_{add_name} "
|
||||
f"-t -d solution_image_{self.solution.id}_{add_name}"
|
||||
)
|
||||
print("run command", run_command)
|
||||
self.call(run_command)
|
||||
|
||||
def notify(self):
|
||||
@ -104,18 +115,39 @@ class BaseTester:
|
||||
f"Задача: {self.solution.task.name}\n"
|
||||
f"Результат: {self.solution.result}\n"
|
||||
f"Очки решения: {Progress.by_solution(self.solution).score}\n"
|
||||
f"Текущий рейтинг: {self.solution.user.userinfo.rating}")
|
||||
f"Текущий рейтинг: {self.solution.user.userinfo.rating}",
|
||||
)
|
||||
|
||||
def cleanup(self):
|
||||
self.solution.save()
|
||||
send_to_queue("cleaner", {"type": "container", "name": f"solution_{self.solution.id}"})
|
||||
send_to_queue(
|
||||
"cleaner", {"type": "container", "name": f"solution_{self.solution.id}"}
|
||||
)
|
||||
if self.checker_code:
|
||||
send_to_queue("cleaner", {"type": "container", "name": f"solution_{self.solution.id}_checker"})
|
||||
send_to_queue(
|
||||
"cleaner",
|
||||
{"type": "container", "name": f"solution_{self.solution.id}_checker"},
|
||||
)
|
||||
for file in self.solution.task.dockerfiles:
|
||||
add_name = file.filename[11:]
|
||||
send_to_queue("cleaner", {"type": "container", "name": f"solution_container_{self.solution.id}_{add_name}"})
|
||||
send_to_queue("cleaner", {"type": "image", "name": f"solution_image_{self.solution.id}_{add_name}"})
|
||||
send_to_queue("cleaner", {"type": "network", "name": f"solution_network_{self.solution.id}"})
|
||||
send_to_queue(
|
||||
"cleaner",
|
||||
{
|
||||
"type": "container",
|
||||
"name": f"solution_container_{self.solution.id}_{add_name}",
|
||||
},
|
||||
)
|
||||
send_to_queue(
|
||||
"cleaner",
|
||||
{
|
||||
"type": "image",
|
||||
"name": f"solution_image_{self.solution.id}_{add_name}",
|
||||
},
|
||||
)
|
||||
send_to_queue(
|
||||
"cleaner",
|
||||
{"type": "network", "name": f"solution_network_{self.solution.id}"},
|
||||
)
|
||||
|
||||
def save_progress(self):
|
||||
progress = Progress.objects.get(
|
||||
@ -130,23 +162,17 @@ class BaseTester:
|
||||
def execute(self):
|
||||
self.solution.result = CONSTS["testing_status"]
|
||||
self.save_solution()
|
||||
with TemporaryDirectory(dir='/tmp') as self.path:
|
||||
with TemporaryDirectory(dir="/tmp") as self.path:
|
||||
for file in self.solution.solutionfiles:
|
||||
dirs = file.path.split("/")
|
||||
for i in range(len(dirs) - 1):
|
||||
name = join(
|
||||
self.path, "/".join(dirs[: i + 1])
|
||||
)
|
||||
name = join(self.path, "/".join(dirs[: i + 1]))
|
||||
if not exists(name):
|
||||
mkdir(name)
|
||||
with open(
|
||||
join(self.path, file.path), "wb"
|
||||
) as fs:
|
||||
with open(join(self.path, file.path), "wb") as fs:
|
||||
fs.write(file.bytes.replace(b"\r\n", b"\n"))
|
||||
for file in self.solution.task.extrafiles:
|
||||
with open(
|
||||
join(self.path, file.filename), 'wb'
|
||||
) as fs:
|
||||
with open(join(self.path, file.filename), "wb") as fs:
|
||||
bts = file.bytes
|
||||
fs.write(bts)
|
||||
print("Files copied")
|
||||
@ -157,26 +183,39 @@ class BaseTester:
|
||||
checker = self.solution.task.checkerfile
|
||||
if checker is not None:
|
||||
self.checker_code = checker.text
|
||||
call(f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id}_checker --volume={self.path}:/app -t -d python:3.6", shell=True)
|
||||
call(
|
||||
f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id}_checker --volume={self.path}:/app -t -d python:3.6",
|
||||
shell=True,
|
||||
)
|
||||
print("Container created")
|
||||
try:
|
||||
self.before_test()
|
||||
print("before test finished")
|
||||
for test in self.solution.task.tests:
|
||||
if not test.filename.endswith(".a"):
|
||||
self.predicted = open(join(self.path, test.filename + '.a'), 'r').read().strip().replace('\r\n', '\n')
|
||||
print('predicted:', self.predicted)
|
||||
self.predicted = (
|
||||
open(join(self.path, test.filename + ".a"), "r")
|
||||
.read()
|
||||
.strip()
|
||||
.replace("\r\n", "\n")
|
||||
)
|
||||
print("predicted:", self.predicted)
|
||||
self.solution.test = int(test.filename)
|
||||
self.solution.extras[test.filename] = {'predicted': self.predicted, 'output': ''}
|
||||
self.solution.extras[test.filename] = {
|
||||
"predicted": self.predicted,
|
||||
"output": "",
|
||||
}
|
||||
self.save_solution()
|
||||
try:
|
||||
self.test(test.filename)
|
||||
finally:
|
||||
if exists(join(self.path, "output.txt")):
|
||||
try:
|
||||
self.solution.extras[test.filename]['output'] = open(join(self.path, 'output.txt'), 'r').read()
|
||||
self.solution.extras[test.filename][
|
||||
"output"
|
||||
] = open(join(self.path, "output.txt"), "r").read()
|
||||
except UnicodeDecodeError:
|
||||
self.solution.extras[test.filename]['output'] = ''
|
||||
self.solution.extras[test.filename]["output"] = ""
|
||||
self.save_solution()
|
||||
self.after_test()
|
||||
self.solution.result = CONSTS["ok_status"]
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import datetime
|
||||
from random import choice
|
||||
from time import sleep
|
||||
|
||||
from django.core.management import BaseCommand
|
||||
from requests import get, post
|
||||
|
||||
from Sprint import settings
|
||||
@ -53,3 +55,15 @@ class Timer:
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.end_time = datetime.datetime.now()
|
||||
self.solution.extras[self.test]['time_spent'] = (self.end_time - self.start_time).total_seconds() * 1000
|
||||
|
||||
|
||||
class LoopWorker(BaseCommand):
|
||||
sleep_period = 5
|
||||
|
||||
def go(self):
|
||||
raise NotImplementedError("Method go should be implemented")
|
||||
|
||||
def handle(self, *args, **options):
|
||||
while True:
|
||||
self.go()
|
||||
sleep(self.sleep_period)
|
||||
|
||||
@ -1,21 +1,15 @@
|
||||
import datetime
|
||||
from time import sleep
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from Checker.models import Checker
|
||||
from SprintLib.utils import LoopWorker
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
class Command(LoopWorker):
|
||||
help = "starts loop"
|
||||
|
||||
def check_checkers(self):
|
||||
def go(self):
|
||||
for checker in Checker.objects.filter(testing_solution__isnull=False, last_request__lt=timezone.now() - datetime.timedelta(seconds=3)):
|
||||
checker.testing_solution.result = 'In queue'
|
||||
checker.testing_solution.save()
|
||||
|
||||
def handle(self, *args, **options):
|
||||
while True:
|
||||
self.check_checkers()
|
||||
sleep(5)
|
||||
@ -1,33 +1,43 @@
|
||||
from subprocess import call
|
||||
from subprocess import call, PIPE, run
|
||||
|
||||
from SprintLib.queue import MessagingSupport, send_to_queue
|
||||
from Main.models import Solution
|
||||
from SprintLib.utils import LoopWorker
|
||||
|
||||
|
||||
class Command(MessagingSupport):
|
||||
class Command(LoopWorker):
|
||||
help = "starts docker cleaner"
|
||||
queue_name = "cleaner"
|
||||
|
||||
def go(self):
|
||||
result = run("docker ps", universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
|
||||
lines = result.stdout.split('\n')[1:]
|
||||
for line in lines:
|
||||
line = [i for i in line.split() if i]
|
||||
if line and line[-1].startswith('solution_'):
|
||||
for el in line[-1].split('_'):
|
||||
if el.isnumeric():
|
||||
solution_id = int(el)
|
||||
break
|
||||
solution = Solution.objects.filter(id=solution_id).first()
|
||||
if solution is not None and (solution.result == 'In queue' or solution.result == 'Testing'):
|
||||
continue
|
||||
call(f"docker rm --force {line[-1]}", shell=True)
|
||||
result = run("docker image ls", universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
|
||||
lines = result.stdout.split('\n')[1:]
|
||||
for line in lines:
|
||||
line = [i for i in line.split() if i]
|
||||
if line and line[0].startswith('solution_'):
|
||||
call("docker image rm " + line[0], shell=True)
|
||||
result = run("docker network ls", universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
|
||||
lines = result.stdout.split('\n')[1:]
|
||||
for line in lines:
|
||||
line = [i for i in line.split() if i]
|
||||
if line and line[1].startswith('solution_'):
|
||||
call("docker network rm " + line[0], shell=True)
|
||||
a = 5
|
||||
a += 1
|
||||
|
||||
def handle(self, *args, **options):
|
||||
call('docker image rm $(docker images -q mathwave/sprint-repo)', shell=True)
|
||||
call('docker rm $(docker ps -qa)', shell=True)
|
||||
print("Old images removed")
|
||||
super().handle(*args, **options)
|
||||
|
||||
def process(self, payload: dict):
|
||||
name = payload['name']
|
||||
type = payload['type']
|
||||
if type == 'network':
|
||||
command = f'docker network rm {name}'
|
||||
elif type == 'container':
|
||||
command = f'docker rm --force {name}'
|
||||
elif type == 'image':
|
||||
command = f'docker image rm --force {name}'
|
||||
else:
|
||||
raise NotImplementedError(f"Unknown type {type}")
|
||||
print(f"Executing command {command}")
|
||||
code = call(command, shell=True)
|
||||
if code == 0:
|
||||
print(f"Removed {type} {name}")
|
||||
else:
|
||||
print("Something went wrong")
|
||||
send_to_queue(self.queue_name, payload)
|
||||
|
||||
@ -199,7 +199,7 @@ services:
|
||||
parallelism: 1
|
||||
order: stop-first
|
||||
|
||||
loop:
|
||||
checker_cleaner:
|
||||
image: mathwave/sprint-repo:sprint
|
||||
networks:
|
||||
- net
|
||||
@ -210,7 +210,7 @@ services:
|
||||
DB_PASSWORD: $DB_PASSWORD
|
||||
DEBUG: $DEBUG
|
||||
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
|
||||
command: ./manage.py loop
|
||||
command: ./manage.py checker_cleaner
|
||||
deploy:
|
||||
mode: replicated
|
||||
restart_policy:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user