From f81029b211072d99292f5a539ddc2b1480c82b0b Mon Sep 17 00:00:00 2001 From: Administrator Date: Sun, 8 May 2022 19:59:06 +0000 Subject: [PATCH] Redis --- .deploy/deploy-dev.yaml | 50 +++++++++++++++++++ .deploy/deploy-prod.yaml | 50 +++++++++++++++++++ FileStorage/views/upload_file.py | 5 +- SprintLib/redis.py | 28 +++++++++++ SprintLib/testers/BaseTester.py | 10 +++- daemons/management/commands/docker_cleaner.py | 40 +++++++++------ daemons/management/commands/redis_worker.py | 29 +++++++++++ 7 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 SprintLib/redis.py create mode 100644 daemons/management/commands/redis_worker.py diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index e59e0ef..9ae4984 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -56,6 +56,20 @@ services: parallelism: 1 order: start-first + redis: + image: redis + networks: + - net + deploy: + mode: replicated + restart_policy: + condition: any + placement: + constraints: [node.role == manager] + update_config: + parallelism: 1 + order: start-first + migrations: image: mathwave/sprint-repo:sprint command: ./manage.py migrate @@ -65,6 +79,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -81,6 +96,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -100,6 +116,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -123,6 +140,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -143,6 +161,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -164,6 +183,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -187,6 +207,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -207,6 +228,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -228,6 +250,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -252,6 +275,30 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" + DB_PASSWORD: $DB_PASSWORD + DEBUG: $DEBUG + TELEGRAM_TOKEN: $TELEGRAM_TOKEN + volumes: + - /var/run/docker.sock:/var/run/docker.sock + deploy: + mode: global + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + + redis_worker: + image: mathwave/sprint-repo:sprint + networks: + - net + command: ./manage.py redis_worker + environment: + DB_HOST: "postgres" + FS_HOST: "storage" + RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -274,6 +321,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -295,6 +343,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -316,6 +365,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index 9166831..a7c2525 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -56,6 +56,20 @@ services: parallelism: 1 order: start-first + redis: + image: redis + networks: + - net + deploy: + mode: replicated + restart_policy: + condition: any + placement: + constraints: [node.role == manager] + update_config: + parallelism: 1 + order: start-first + migrations: image: mathwave/sprint-repo:sprint command: ./manage.py migrate @@ -65,6 +79,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -81,6 +96,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -100,6 +116,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -123,6 +140,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -143,6 +161,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -164,6 +183,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -187,6 +207,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -207,6 +228,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -228,6 +250,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -252,6 +275,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -265,6 +289,29 @@ services: parallelism: 1 order: start-first + redis_worker: + image: mathwave/sprint-repo:sprint + networks: + - net + command: ./manage.py redis_worker + environment: + DB_HOST: "postgres" + FS_HOST: "storage" + RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" + DB_PASSWORD: $DB_PASSWORD + DEBUG: $DEBUG + TELEGRAM_TOKEN: $TELEGRAM_TOKEN + volumes: + - /var/run/docker.sock:/var/run/docker.sock + deploy: + mode: global + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + file_generator: image: mathwave/sprint-repo:sprint networks: @@ -274,6 +321,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -295,6 +343,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN @@ -316,6 +365,7 @@ services: DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" + REDIS_HOST: "redis" DB_PASSWORD: $DB_PASSWORD DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN diff --git a/FileStorage/views/upload_file.py b/FileStorage/views/upload_file.py index 2795c2e..d0a2593 100644 --- a/FileStorage/views/upload_file.py +++ b/FileStorage/views/upload_file.py @@ -1,11 +1,12 @@ -import os - from aiohttp import web from FileStorage.sync import write_meta import aiofiles +from SprintLib.redis import lock + +@lock() async def upload_file(request): file_id = await write_meta(request) async with aiofiles.open("data/" + str(file_id), "wb") as fs: diff --git a/SprintLib/redis.py b/SprintLib/redis.py new file mode 100644 index 0000000..7e367dc --- /dev/null +++ b/SprintLib/redis.py @@ -0,0 +1,28 @@ +import os + +import redis + + +def get_redis(): + return redis.Redis(host=os.getenv("REDIS_HOST", "127.0.0.1")) + + +def get(key): + with get_redis() as r: + return r.get(key) + + +def set(key, value): + with get_redis() as r: + return r.set(key, value) + + +def lock(name='lock'): + def dec(fun): + def wrapper(*args, **kwargs): + with get_redis() as r: + with r.lock(name): + return fun(*args, **kwargs) + + return wrapper + return dec diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 071ca38..8c38349 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory from Main.models.progress import Progress from Sprint.settings import CONSTS -from SprintLib.queue import notify +from SprintLib.queue import notify, send_to_queue from SprintLib.utils import Timer @@ -132,6 +132,14 @@ class BaseTester: def cleanup(self): self.solution.save() + send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_{self.solution.id}"}) + if self.checker_code: + send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_{self.solution.id}_checker"}) + for file in self.solution.task.dockerfiles: + add_name = file.filename[11:] + send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_container_{self.solution.id}_{add_name}"}) + send_to_queue("redis", {"action": "docker", "key": "images", "value": f"solution_image_{self.solution.id}_{add_name}"}) + send_to_queue("redis", {"action": "docker", "key": "networks", "value": f"solution_network_{self.solution.id}"}) def save_progress(self): progress = Progress.objects.get( diff --git a/daemons/management/commands/docker_cleaner.py b/daemons/management/commands/docker_cleaner.py index f644eec..cb06e93 100644 --- a/daemons/management/commands/docker_cleaner.py +++ b/daemons/management/commands/docker_cleaner.py @@ -1,27 +1,37 @@ from subprocess import call -from django.db.models import Q - -from Main.models import Solution +from SprintLib.redis import lock, get_redis from SprintLib.utils import LoopWorker class Command(LoopWorker): help = "starts docker cleaner" + @lock("docker") def go(self): - for solution in Solution.objects.filter(~Q(result="Testing") | ~Q(result="In queue"), docker_instances__isnull=False): - for instance in sorted(solution.docker_instances, key=lambda x: x['type']): - if instance['type'] == 'network': - call(f"docker network rm {instance['name']}", shell=True) - elif instance['type'] == 'image': - call(f"docker image rm --force {instance['name']}", shell=True) - elif instance['type'] == 'container': - call(f"docker rm --force {instance['name']}", shell=True) - else: - raise ValueError(f"Unknown docker type {instance['type']}") - solution.docker_instances = None - solution.save() + containers, images, networks = list(), list(), list() + with get_redis() as r: + while r.llen("containers") != 0: + value = r.rpop("containers").decode("utf-8") + return_code = call(f"docker rm --force {value}", shell=True) + if return_code != 0: + containers.append(value) + while r.llen("images") != 0: + value = r.rpop("images").decode("utf-8") + return_code = call(f"docker image rm --force {value}", shell=True) + if return_code != 0: + images.append(value) + while r.llen("networks") != 0: + value = r.rpop("networks").decode("utf-8") + return_code = call(f"docker network rm {value}", shell=True) + if return_code != 0: + networks.append(value) + if containers: + r.lpush("containers", *containers) + if images: + r.lpush("images", *images) + if networks: + r.lpush("networks", *networks) def handle(self, *args, **options): call('docker image rm $(docker images -q mathwave/sprint-repo)', shell=True) diff --git a/daemons/management/commands/redis_worker.py b/daemons/management/commands/redis_worker.py new file mode 100644 index 0000000..3b0359f --- /dev/null +++ b/daemons/management/commands/redis_worker.py @@ -0,0 +1,29 @@ +from SprintLib.queue import MessagingSupport +from SprintLib.redis import lock, get_redis + + +class Command(MessagingSupport): + help = "Starts redis worker" + queue_name = "redis" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.handlers = { + "docker": self.docker_handler + } + + @lock("docker") + def docker_handler(self, payload): + key = payload["key"] + value = payload["value"] + with get_redis() as r: + r.lpush(key, value) + + def process(self, payload: dict): + action = payload.get("action") + if action is None: + raise ValueError("No action field in message") + handler = self.handlers.get(action) + if handler is None: + raise ValueError(f"No handler for action: {action}") + handler(payload)