self/sprint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'redis' into 'master'

Browse Source 
Redis

See merge request root/sprint!2
This commit is contained in:
Administrator 2022-05-08 19:59:06 +00:00
commit bfd5ecc0bb
7 changed files with 194 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
.deploy/deploy-dev.yaml
View File

@ -56,6 +56,20 @@ services:
parallelism: 1
order: start-first
redis:
image: redis
networks:
- net
deploy:
mode: replicated
restart_policy:
condition: any
placement:
constraints: [node.role == manager]
update_config:
parallelism: 1
order: start-first
migrations:
image: mathwave/sprint-repo:sprint
command: ./manage.py migrate
@ -65,6 +79,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -81,6 +96,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -100,6 +116,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -123,6 +140,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -143,6 +161,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -164,6 +183,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -187,6 +207,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -207,6 +228,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -228,6 +250,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -252,6 +275,30 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
volumes:
- /var/run/docker.sock:/var/run/docker.sock
deploy:
mode: global
restart_policy:
condition: any
update_config:
parallelism: 1
order: start-first
redis_worker:
image: mathwave/sprint-repo:sprint
networks:
- net
command: ./manage.py redis_worker
environment:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -274,6 +321,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -295,6 +343,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -316,6 +365,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN

50
.deploy/deploy-prod.yaml
View File

@ -56,6 +56,20 @@ services:
parallelism: 1
order: start-first
redis:
image: redis
networks:
- net
deploy:
mode: replicated
restart_policy:
condition: any
placement:
constraints: [node.role == manager]
update_config:
parallelism: 1
order: start-first
migrations:
image: mathwave/sprint-repo:sprint
command: ./manage.py migrate
@ -65,6 +79,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -81,6 +96,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -100,6 +116,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -123,6 +140,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -143,6 +161,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -164,6 +183,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -187,6 +207,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -207,6 +228,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -228,6 +250,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -252,6 +275,30 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
volumes:
- /var/run/docker.sock:/var/run/docker.sock
deploy:
mode: global
restart_policy:
condition: any
update_config:
parallelism: 1
order: start-first
redis_worker:
image: mathwave/sprint-repo:sprint
networks:
- net
command: ./manage.py redis_worker
environment:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -274,6 +321,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -295,6 +343,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN
@ -316,6 +365,7 @@ services:
DB_HOST: "postgres"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq"
REDIS_HOST: "redis"
DB_PASSWORD: $DB_PASSWORD
DEBUG: $DEBUG
TELEGRAM_TOKEN: $TELEGRAM_TOKEN

5
FileStorage/views/upload_file.py
View File

@ -1,11 +1,12 @@
import os
from aiohttp import web
from FileStorage.sync import write_meta
import aiofiles
from SprintLib.redis import lock
@lock()
async def upload_file(request):
file_id = await write_meta(request)
async with aiofiles.open("data/" + str(file_id), "wb") as fs:

28
SprintLib/redis.py Normal file
View File

@ -0,0 +1,28 @@
import os
import redis
def get_redis():
return redis.Redis(host=os.getenv("REDIS_HOST", "127.0.0.1"))
def get(key):
with get_redis() as r:
return r.get(key)
def set(key, value):
with get_redis() as r:
return r.set(key, value)
def lock(name='lock'):
def dec(fun):
def wrapper(*args, **kwargs):
with get_redis() as r:
with r.lock(name):
return fun(*args, **kwargs)
return wrapper
return dec

10
SprintLib/testers/BaseTester.py
View File

@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory
from Main.models.progress import Progress
from Sprint.settings import CONSTS
from SprintLib.queue import notify
from SprintLib.queue import notify, send_to_queue
from SprintLib.utils import Timer
@ -132,6 +132,14 @@ class BaseTester:
def cleanup(self):
self.solution.save()
send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_{self.solution.id}"})
if self.checker_code:
send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_{self.solution.id}_checker"})
for file in self.solution.task.dockerfiles:
add_name = file.filename[11:]
send_to_queue("redis", {"action": "docker", "key": "containers", "value": f"solution_container_{self.solution.id}_{add_name}"})
send_to_queue("redis", {"action": "docker", "key": "images", "value": f"solution_image_{self.solution.id}_{add_name}"})
send_to_queue("redis", {"action": "docker", "key": "networks", "value": f"solution_network_{self.solution.id}"})
def save_progress(self):
progress = Progress.objects.get(

40
daemons/management/commands/docker_cleaner.py
View File

@ -1,27 +1,37 @@
from subprocess import call
from django.db.models import Q
from Main.models import Solution
from SprintLib.redis import lock, get_redis
from SprintLib.utils import LoopWorker
class Command(LoopWorker):
help = "starts docker cleaner"
@lock("docker")
def go(self):
for solution in Solution.objects.filter(~Q(result="Testing") | ~Q(result="In queue"), docker_instances__isnull=False):
for instance in sorted(solution.docker_instances, key=lambda x: x['type']):
if instance['type'] == 'network':
call(f"docker network rm {instance['name']}", shell=True)
elif instance['type'] == 'image':
call(f"docker image rm --force {instance['name']}", shell=True)
elif instance['type'] == 'container':
call(f"docker rm --force {instance['name']}", shell=True)
else:
raise ValueError(f"Unknown docker type {instance['type']}")
solution.docker_instances = None
solution.save()
containers, images, networks = list(), list(), list()
with get_redis() as r:
while r.llen("containers") != 0:
value = r.rpop("containers").decode("utf-8")
return_code = call(f"docker rm --force {value}", shell=True)
if return_code != 0:
containers.append(value)
while r.llen("images") != 0:
value = r.rpop("images").decode("utf-8")
return_code = call(f"docker image rm --force {value}", shell=True)
if return_code != 0:
images.append(value)
while r.llen("networks") != 0:
value = r.rpop("networks").decode("utf-8")
return_code = call(f"docker network rm {value}", shell=True)
if return_code != 0:
networks.append(value)
if containers:
r.lpush("containers", *containers)
if images:
r.lpush("images", *images)
if networks:
r.lpush("networks", *networks)
def handle(self, *args, **options):
call('docker image rm $(docker images -q mathwave/sprint-repo)', shell=True)

29
daemons/management/commands/redis_worker.py Normal file
View File

@ -0,0 +1,29 @@
from SprintLib.queue import MessagingSupport
from SprintLib.redis import lock, get_redis
class Command(MessagingSupport):
help = "Starts redis worker"
queue_name = "redis"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.handlers = {
"docker": self.docker_handler
}
@lock("docker")
def docker_handler(self, payload):
key = payload["key"]
value = payload["value"]
with get_redis() as r:
r.lpush(key, value)
def process(self, payload: dict):
action = payload.get("action")
if action is None:
raise ValueError("No action field in message")
handler = self.handlers.get(action)
if handler is None:
raise ValueError(f"No handler for action: {action}")
handler(payload)