Merge branch 'redis' into 'master'

Redis

See merge request root/sprint!2

.deploy/deploy-dev.yaml

@@ -56,6 +56,20 @@ services:
         parallelism: 1
         order: start-first
 
+  redis:
+    image: redis
+    networks:
+      - net
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      placement:
+        constraints: [node.role == manager]
+      update_config:
+        parallelism: 1
+        order: start-first
+
   migrations:
     image: mathwave/sprint-repo:sprint
     command: ./manage.py migrate
@@ -65,6 +79,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -81,6 +96,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -100,6 +116,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -123,6 +140,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -143,6 +161,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -164,6 +183,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -187,6 +207,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -207,6 +228,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -228,6 +250,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -252,6 +275,30 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
+      DB_PASSWORD: $DB_PASSWORD
+      DEBUG: $DEBUG
+      TELEGRAM_TOKEN: $TELEGRAM_TOKEN
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+      update_config:
+        parallelism: 1
+        order: start-first
+
+  redis_worker:
+    image: mathwave/sprint-repo:sprint
+    networks:
+      - net
+    command: ./manage.py redis_worker
+    environment:
+      DB_HOST: "postgres"
+      FS_HOST: "storage"
+      RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -274,6 +321,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -295,6 +343,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -316,6 +365,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN

.deploy/deploy-prod.yaml

@@ -56,6 +56,20 @@ services:
         parallelism: 1
         order: start-first
 
+  redis:
+    image: redis
+    networks:
+      - net
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      placement:
+        constraints: [node.role == manager]
+      update_config:
+        parallelism: 1
+        order: start-first
+
   migrations:
     image: mathwave/sprint-repo:sprint
     command: ./manage.py migrate
@@ -65,6 +79,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -81,6 +96,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -100,6 +116,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -123,6 +140,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -143,6 +161,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -164,6 +183,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -187,6 +207,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -207,6 +228,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -228,6 +250,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -252,6 +275,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -265,6 +289,29 @@ services:
         parallelism: 1
         order: start-first
 
+  redis_worker:
+    image: mathwave/sprint-repo:sprint
+      networks:
+        - net
+      command: ./manage.py redis_worker
+      environment:
+        DB_HOST: "postgres"
+        FS_HOST: "storage"
+        RABBIT_HOST: "rabbitmq"
+        REDIS_HOST: "redis"
+        DB_PASSWORD: $DB_PASSWORD
+        DEBUG: $DEBUG
+        TELEGRAM_TOKEN: $TELEGRAM_TOKEN
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+      deploy:
+        mode: global
+        restart_policy:
+          condition: any
+        update_config:
+          parallelism: 1
+          order: start-first
+
   file_generator:
     image: mathwave/sprint-repo:sprint
     networks:
@@ -274,6 +321,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -295,6 +343,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@@ -316,6 +365,7 @@ services:
       DB_HOST: "postgres"
       FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq"
+      REDIS_HOST: "redis"
       DB_PASSWORD: $DB_PASSWORD
       DEBUG: $DEBUG
       TELEGRAM_TOKEN: $TELEGRAM_TOKEN

FileStorage/views/upload_file.py

@@ -1,11 +1,12 @@
-import os
-
 from aiohttp import web
 
 from FileStorage.sync import write_meta
 import aiofiles
 
+from SprintLib.redis import lock
 
+
+@lock()
 async def upload_file(request):
     file_id = await write_meta(request)
     async with aiofiles.open("data/" + str(file_id), "wb") as fs:

SprintLib/redis.py

@@ -0,0 +1,28 @@
+import os
+
+import redis
+
+
+def get_redis():
+    return redis.Redis(host=os.getenv("REDIS_HOST", "127.0.0.1"))
+
+
+def get(key):
+    with get_redis() as r:
+        return r.get(key)
+
+
+def set(key, value):
+    with get_redis() as r:
+        return r.set(key, value)
+
+
+def lock(name='lock'):
+    def dec(fun):
+        def wrapper(*args, **kwargs):
+            with get_redis() as r:
+                with r.lock(name):
+                    return fun(*args, **kwargs)
+
+        return wrapper
+    return dec

SprintLib/testers/BaseTester.py

@@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory
 
 from Main.models.progress import Progress
 from Sprint.settings import CONSTS
-from SprintLib.queue import notify
+from SprintLib.queue import notify, send_to_queue
 from SprintLib.utils import Timer
 
 
@@ -132,6 +132,14 @@ class BaseTester:
 
     def cleanup(self):
         self.solution.save()
+        send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_{self.solution.id}"})
+        if self.checker_code:
+            send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_{self.solution.id}_checker"})
+        for file in self.solution.task.dockerfiles:
+            add_name = file.filename[11:]
+            send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_container_{self.solution.id}_{add_name}"})
+            send_to_queue("redis", {"action": "docker", "key": "images", "value": f"solution_image_{self.solution.id}_{add_name}"})
+        send_to_queue("redis", {"action": "docker", "key": "networks", "value": f"solution_network_{self.solution.id}"})
 
     def save_progress(self):
         progress = Progress.objects.get(

daemons/management/commands/docker_cleaner.py

@@ -1,27 +1,37 @@
 from subprocess import call
 
-from django.db.models import Q
+from SprintLib.redis import lock, get_redis
-
-from Main.models import Solution
 from SprintLib.utils import LoopWorker
 
 
 class Command(LoopWorker):
     help = "starts docker cleaner"
 
+    @lock("docker")
     def go(self):
-        for solution in Solution.objects.filter(~Q(result="Testing") | ~Q(result="In queue"), docker_instances__isnull=False):
+        containers, images, networks = list(), list(), list()
-            for instance in sorted(solution.docker_instances, key=lambda x: x['type']):
+        with get_redis() as r:
-                if instance['type'] == 'network':
+            while r.llen("containers") != 0:
-                    call(f"docker network rm {instance['name']}", shell=True)
+                value = r.rpop("containers").decode("utf-8")
-                elif instance['type'] == 'image':
+                return_code = call(f"docker rm --force {value}", shell=True)
-                    call(f"docker image rm --force {instance['name']}", shell=True)
+                if return_code != 0:
-                elif instance['type'] == 'container':
+                    containers.append(value)
-                    call(f"docker rm --force {instance['name']}", shell=True)
+            while r.llen("images") != 0:
-                else:
+                value = r.rpop("images").decode("utf-8")
-                    raise ValueError(f"Unknown docker type {instance['type']}")
+                return_code = call(f"docker image rm --force {value}", shell=True)
-            solution.docker_instances = None
+                if return_code != 0:
-            solution.save()
+                    images.append(value)
+            while r.llen("networks") != 0:
+                value = r.rpop("networks").decode("utf-8")
+                return_code = call(f"docker network rm {value}", shell=True)
+                if return_code != 0:
+                    networks.append(value)
+            if containers:
+                r.lpush("containers", *containers)
+            if images:
+                r.lpush("images", *images)
+            if networks:
+                r.lpush("networks", *networks)
 
     def handle(self, *args, **options):
         call('docker image rm $(docker images -q mathwave/sprint-repo)', shell=True)

daemons/management/commands/redis_worker.py

@@ -0,0 +1,29 @@
+from SprintLib.queue import MessagingSupport
+from SprintLib.redis import lock, get_redis
+
+
+class Command(MessagingSupport):
+    help = "Starts redis worker"
+    queue_name = "redis"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.handlers = {
+            "docker": self.docker_handler
+        }
+
+    @lock("docker")
+    def docker_handler(self, payload):
+        key = payload["key"]
+        value = payload["value"]
+        with get_redis() as r:
+            r.lpush(key, value)
+
+    def process(self, payload: dict):
+        action = payload.get("action")
+        if action is None:
+            raise ValueError("No action field in message")
+        handler = self.handlers.get(action)
+        if handler is None:
+            raise ValueError(f"No handler for action: {action}")
+        handler(payload)