docker cleaner

SprintLib/testers/BaseTester.py

@@ -39,23 +39,32 @@ class BaseTester:
             )
         if code != 0:
             raise TestException("RE")
-        result = open(join(self.path, "output.txt"), "r").read().strip().replace('\r\n', '\n')
+        result = (
+            open(join(self.path, "output.txt"), "r")
+            .read()
+            .strip()
+            .replace("\r\n", "\n")
+        )
         print("got result", result)
         if self.checker_code is not None:
-            print('using checker')
+            print("using checker")
-            with open(join(self.path, 'expected.txt'), 'w') as fs:
+            with open(join(self.path, "expected.txt"), "w") as fs:
                 fs.write(self.predicted)
-            with open(join(self.path, 'checker.py'), 'w') as fs:
+            with open(join(self.path, "checker.py"), "w") as fs:
                 fs.write(self.checker_code)
-            code = call(f'docker exec -i solution_{self.solution.id}_checker sh -c "cd app && python checker.py"', shell=True, timeout=1)
+            code = call(
+                f'docker exec -i solution_{self.solution.id}_checker sh -c "cd app && python checker.py"',
+                shell=True,
+                timeout=1,
+            )
             if code != 0:
                 raise TestException("WA")
         else:
-            print('using simple check')
+            print("using simple check")
             if result != self.predicted:
-                print('incorrect')
+                print("incorrect")
                 raise TestException("WA")
-            print('correct')
+            print("correct")
 
     def after_test(self):
         pass
@@ -71,7 +80,7 @@ class BaseTester:
     def call(self, command):
         print(f"Executing command: {command}")
         if exists(self.path):
-            return call(f'cd {self.path} && {command}', shell=True)
+            return call(f"cd {self.path} && {command}", shell=True)
         else:
             return call(command, shell=True)
 
@@ -85,15 +94,17 @@ class BaseTester:
         self.call(f"docker network create solution_network_{self.solution.id}")
         for file in self.solution.task.dockerfiles:
             add_name = file.filename[11:]
-            with open(join(self.path, 'Dockerfile'), 'w') as fs:
+            with open(join(self.path, "Dockerfile"), "w") as fs:
                 fs.write(file.text)
             self.call(f"docker build -t solution_image_{self.solution.id}_{add_name} .")
-            run_command = f"docker run "\
+            run_command = (
-                          f"--hostname {add_name} "\
+                f"docker run "
-                          f"--network solution_network_{self.solution.id} "\
+                f"--hostname {add_name} "
-                          f"--name solution_container_{self.solution.id}_{add_name} "\
+                f"--network solution_network_{self.solution.id} "
-                          f"-t -d solution_image_{self.solution.id}_{add_name}"
+                f"--name solution_container_{self.solution.id}_{add_name} "
-            print('run command', run_command)
+                f"-t -d solution_image_{self.solution.id}_{add_name}"
+            )
+            print("run command", run_command)
             self.call(run_command)
 
     def notify(self):
@@ -104,18 +115,39 @@ class BaseTester:
             f"Задача: {self.solution.task.name}\n"
             f"Результат: {self.solution.result}\n"
             f"Очки решения: {Progress.by_solution(self.solution).score}\n"
-            f"Текущий рейтинг: {self.solution.user.userinfo.rating}")
+            f"Текущий рейтинг: {self.solution.user.userinfo.rating}",
+        )
 
     def cleanup(self):
         self.solution.save()
-        send_to_queue("cleaner", {"type": "container", "name": f"solution_{self.solution.id}"})
+        send_to_queue(
+            "cleaner", {"type": "container", "name": f"solution_{self.solution.id}"}
+        )
         if self.checker_code:
-            send_to_queue("cleaner", {"type": "container", "name": f"solution_{self.solution.id}_checker"})
+            send_to_queue(
+                "cleaner",
+                {"type": "container", "name": f"solution_{self.solution.id}_checker"},
+            )
         for file in self.solution.task.dockerfiles:
             add_name = file.filename[11:]
-            send_to_queue("cleaner", {"type": "container", "name": f"solution_container_{self.solution.id}_{add_name}"})
+            send_to_queue(
-            send_to_queue("cleaner", {"type": "image", "name": f"solution_image_{self.solution.id}_{add_name}"})
+                "cleaner",
-        send_to_queue("cleaner", {"type": "network", "name": f"solution_network_{self.solution.id}"})
+                {
+                    "type": "container",
+                    "name": f"solution_container_{self.solution.id}_{add_name}",
+                },
+            )
+            send_to_queue(
+                "cleaner",
+                {
+                    "type": "image",
+                    "name": f"solution_image_{self.solution.id}_{add_name}",
+                },
+            )
+        send_to_queue(
+            "cleaner",
+            {"type": "network", "name": f"solution_network_{self.solution.id}"},
+        )
 
     def save_progress(self):
         progress = Progress.objects.get(
@@ -130,23 +162,17 @@ class BaseTester:
     def execute(self):
         self.solution.result = CONSTS["testing_status"]
         self.save_solution()
-        with TemporaryDirectory(dir='/tmp') as self.path:
+        with TemporaryDirectory(dir="/tmp") as self.path:
             for file in self.solution.solutionfiles:
                 dirs = file.path.split("/")
                 for i in range(len(dirs) - 1):
-                    name = join(
+                    name = join(self.path, "/".join(dirs[: i + 1]))
-                        self.path, "/".join(dirs[: i + 1])
-                    )
                     if not exists(name):
                         mkdir(name)
-                with open(
+                with open(join(self.path, file.path), "wb") as fs:
-                    join(self.path, file.path), "wb"
-                ) as fs:
                     fs.write(file.bytes.replace(b"\r\n", b"\n"))
             for file in self.solution.task.extrafiles:
-                with open(
+                with open(join(self.path, file.filename), "wb") as fs:
-                    join(self.path, file.filename), 'wb'
-                ) as fs:
                     bts = file.bytes
                     fs.write(bts)
             print("Files copied")
@@ -157,26 +183,39 @@ class BaseTester:
             checker = self.solution.task.checkerfile
             if checker is not None:
                 self.checker_code = checker.text
-                call(f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id}_checker --volume={self.path}:/app -t -d python:3.6", shell=True)
+                call(
+                    f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id}_checker --volume={self.path}:/app -t -d python:3.6",
+                    shell=True,
+                )
             print("Container created")
             try:
                 self.before_test()
                 print("before test finished")
                 for test in self.solution.task.tests:
                     if not test.filename.endswith(".a"):
-                        self.predicted = open(join(self.path, test.filename + '.a'), 'r').read().strip().replace('\r\n', '\n')
+                        self.predicted = (
-                        print('predicted:', self.predicted)
+                            open(join(self.path, test.filename + ".a"), "r")
+                            .read()
+                            .strip()
+                            .replace("\r\n", "\n")
+                        )
+                        print("predicted:", self.predicted)
                         self.solution.test = int(test.filename)
-                        self.solution.extras[test.filename] = {'predicted': self.predicted, 'output': ''}
+                        self.solution.extras[test.filename] = {
+                            "predicted": self.predicted,
+                            "output": "",
+                        }
                         self.save_solution()
                         try:
                             self.test(test.filename)
                         finally:
                             if exists(join(self.path, "output.txt")):
                                 try:
-                                    self.solution.extras[test.filename]['output'] = open(join(self.path, 'output.txt'), 'r').read()
+                                    self.solution.extras[test.filename][
+                                        "output"
+                                    ] = open(join(self.path, "output.txt"), "r").read()
                                 except UnicodeDecodeError:
-                                    self.solution.extras[test.filename]['output'] = ''
+                                    self.solution.extras[test.filename]["output"] = ""
                             self.save_solution()
                 self.after_test()
                 self.solution.result = CONSTS["ok_status"]

SprintLib/utils.py

@@ -1,6 +1,8 @@
 import datetime
 from random import choice
+from time import sleep
 
+from django.core.management import BaseCommand
 from requests import get, post
 
 from Sprint import settings
@@ -53,3 +55,15 @@ class Timer:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.end_time = datetime.datetime.now()
         self.solution.extras[self.test]['time_spent'] = (self.end_time - self.start_time).total_seconds() * 1000
+
+
+class LoopWorker(BaseCommand):
+    sleep_period = 5
+
+    def go(self):
+        raise NotImplementedError("Method go should be implemented")
+
+    def handle(self, *args, **options):
+        while True:
+            self.go()
+            sleep(self.sleep_period)

daemons/management/commands/checker_cleaner.py

@@ -1,21 +1,15 @@
 import datetime
-from time import sleep
 
-from django.core.management.base import BaseCommand
 from django.utils import timezone
 
 from Checker.models import Checker
+from SprintLib.utils import LoopWorker
 
 
-class Command(BaseCommand):
+class Command(LoopWorker):
     help = "starts loop"
 
-    def check_checkers(self):
+    def go(self):
         for checker in Checker.objects.filter(testing_solution__isnull=False, last_request__lt=timezone.now() - datetime.timedelta(seconds=3)):
             checker.testing_solution.result = 'In queue'
             checker.testing_solution.save()
-
-    def handle(self, *args, **options):
-        while True:
-            self.check_checkers()
-            sleep(5)

daemons/management/commands/docker_cleaner.py

@@ -1,33 +1,43 @@
-from subprocess import call
+from subprocess import call, PIPE, run
 
-from SprintLib.queue import MessagingSupport, send_to_queue
+from Main.models import Solution
+from SprintLib.utils import LoopWorker
 
 
-class Command(MessagingSupport):
+class Command(LoopWorker):
     help = "starts docker cleaner"
-    queue_name = "cleaner"
+
+    def go(self):
+        result = run("docker ps", universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
+        lines = result.stdout.split('\n')[1:]
+        for line in lines:
+            line = [i for i in line.split() if i]
+            if line and line[-1].startswith('solution_'):
+                for el in line[-1].split('_'):
+                    if el.isnumeric():
+                        solution_id = int(el)
+                        break
+                solution = Solution.objects.filter(id=solution_id).first()
+                if solution is not None and (solution.result == 'In queue' or solution.result == 'Testing'):
+                    continue
+                call(f"docker rm --force {line[-1]}", shell=True)
+        result = run("docker image ls", universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
+        lines = result.stdout.split('\n')[1:]
+        for line in lines:
+            line = [i for i in line.split() if i]
+            if line and line[0].startswith('solution_'):
+                call("docker image rm " + line[0], shell=True)
+        result = run("docker network ls", universal_newlines=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
+        lines = result.stdout.split('\n')[1:]
+        for line in lines:
+            line = [i for i in line.split() if i]
+            if line and line[1].startswith('solution_'):
+                call("docker network rm " + line[0], shell=True)
+        a = 5
+        a += 1
 
     def handle(self, *args, **options):
         call('docker image rm $(docker images -q mathwave/sprint-repo)', shell=True)
-        call('docker rm $(docker ps -qa)', shell=True)
         print("Old images removed")
         super().handle(*args, **options)
 
-    def process(self, payload: dict):
-        name = payload['name']
-        type = payload['type']
-        if type == 'network':
-            command = f'docker network rm {name}'
-        elif type == 'container':
-            command = f'docker rm --force {name}'
-        elif type == 'image':
-            command = f'docker image rm --force {name}'
-        else:
-            raise NotImplementedError(f"Unknown type {type}")
-        print(f"Executing command {command}")
-        code = call(command, shell=True)
-        if code == 0:
-            print(f"Removed {type} {name}")
-        else:
-            print("Something went wrong")
-            send_to_queue(self.queue_name, payload)

docker-compose-deploy.yaml

@@ -199,7 +199,7 @@ services:
         parallelism: 1
         order: stop-first
 
-  loop:
+  checker_cleaner:
     image: mathwave/sprint-repo:sprint
     networks:
       - net
@@ -210,7 +210,7 @@ services:
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
-    command: ./manage.py loop
+    command: ./manage.py checker_cleaner
     deploy:
       mode: replicated
       restart_policy: