diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index 33f1558..cf2bc6c 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -27,9 +27,10 @@ services: - net environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV @@ -51,9 +52,10 @@ services: - net environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV @@ -68,42 +70,16 @@ services: parallelism: 1 order: start-first - storage: - image: mathwave/sprint-repo:sprint - networks: - - net - command: ./manage.py storage - environment: - DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" - RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" - REDIS_HOST: "redis.develop.sprinthub.ru" - RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV - REDIS_PASSWORD: $REDIS_PASSWORD_DEV - DB_PASSWORD: $DB_PASSWORD_DEV - DEBUG: "true" - TELEGRAM_TOKEN: $TELEGRAM_TOKEN_DEV - volumes: - - /sprint-data/data:/usr/src/app/data - deploy: - mode: replicated - restart_policy: - condition: any - placement: - constraints: [node.role == manager] - update_config: - parallelism: 1 - order: start-first - bot: image: mathwave/sprint-repo:sprint networks: - net environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV @@ -124,9 +100,10 @@ services: - net environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV @@ -148,9 +125,10 @@ services: command: ./manage.py receive environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV @@ -175,9 +153,10 @@ services: command: ./manage.py file_generator environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV @@ -199,9 +178,10 @@ services: command: ./manage.py notification_manager environment: DB_HOST: "pg.develop.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.develop.sprinthub.ru" REDIS_HOST: "redis.develop.sprinthub.ru" + MINIO_HOST: "minio.develop.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV REDIS_PASSWORD: $REDIS_PASSWORD_DEV DB_PASSWORD: $DB_PASSWORD_DEV diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index b473873..c2adba2 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -21,43 +21,15 @@ services: parallelism: 1 order: start-first - storage: - image: mathwave/sprint-repo:sprint - networks: - - net - command: ./manage.py storage - environment: - DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" - RABBIT_HOST: "rabbitmq.sprinthub.ru" - REDIS_HOST: "redis.sprinthub.ru" - RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD - REDIS_PASSWORD: $REDIS_PASSWORD_PROD - DB_PASSWORD: $DB_PASSWORD_PROD - DEBUG: "false" - TELEGRAM_TOKEN: $TELEGRAM_TOKEN_PROD - SENTRY_TOKEN: $SENTRY_TOKEN - volumes: - - /sprint-data/data:/usr/src/app/data - deploy: - mode: replicated - restart_policy: - condition: any - placement: - constraints: [node.role == manager] - update_config: - parallelism: 1 - order: start-first - web: image: mathwave/sprint-repo:sprint networks: - net environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD @@ -79,9 +51,9 @@ services: - net environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD @@ -104,9 +76,9 @@ services: - net environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD @@ -129,9 +101,9 @@ services: - net environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD @@ -154,9 +126,10 @@ services: command: ./manage.py receive environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD @@ -182,9 +155,10 @@ services: command: ./manage.py file_generator environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD @@ -207,9 +181,10 @@ services: command: ./manage.py notification_manager environment: DB_HOST: "pg.sprinthub.ru" - FS_HOST: "storage" RABBIT_HOST: "rabbitmq.sprinthub.ru" REDIS_HOST: "redis.sprinthub.ru" + MINIO_HOST: "minio.sprinthub.ru" + MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD REDIS_PASSWORD: $REDIS_PASSWORD_PROD DB_PASSWORD: $DB_PASSWORD_PROD diff --git a/Checker/views.py b/Checker/views.py index b0dfa6b..c7463b1 100644 --- a/Checker/views.py +++ b/Checker/views.py @@ -8,9 +8,9 @@ from django.http import JsonResponse, HttpResponse from django.utils import timezone from Checker.models import Checker -from FileStorage.sync import synchronized_method from Main.models import Solution, SolutionFile, ExtraFile, Progress from SprintLib.queue import send_to_queue +from SprintLib.redis import lock from SprintLib.utils import generate_token @@ -37,7 +37,7 @@ def status(request): return JsonResponse({"status": "incorrect token"}, status=403) -@synchronized_method +@lock('checker') def available(request): try: checker = Checker.objects.get(dynamic_token=request.GET['token']) diff --git a/FileStorage/__init__.py b/FileStorage/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/FileStorage/root.py b/FileStorage/root.py deleted file mode 100644 index 94efa2f..0000000 --- a/FileStorage/root.py +++ /dev/null @@ -1,21 +0,0 @@ -from os import mkdir -from os.path import exists - -from aiohttp import web - -from FileStorage.routes import setup_routes - - -def runserver(): - app = web.Application() - setup_routes(app) - if not exists("data"): - mkdir("data") - if not exists("data/meta.txt"): - with open("data/meta.txt", "w") as fs: - fs.write("0") - web.run_app(app, host="0.0.0.0", port=5555) - - -if __name__ == "__main__": - runserver() diff --git a/FileStorage/routes.py b/FileStorage/routes.py deleted file mode 100644 index 3ce9fcf..0000000 --- a/FileStorage/routes.py +++ /dev/null @@ -1,9 +0,0 @@ -from aiohttp import web - -from FileStorage.views import get_file, upload_file, delete_file - - -def setup_routes(app: web.Application): - app.router.add_get("/get_file", get_file) - app.router.add_post("/upload_file", upload_file) - app.router.add_post("/delete_file", delete_file) diff --git a/FileStorage/sync.py b/FileStorage/sync.py deleted file mode 100644 index fd82ce3..0000000 --- a/FileStorage/sync.py +++ /dev/null @@ -1,26 +0,0 @@ -import threading - -import aiofiles - - -def synchronized_method(method): - outer_lock = threading.Lock() - lock_name = "__" + method.__name__ + "_lock" + "__" - - def sync_method(self, *args, **kws): - with outer_lock: - if not hasattr(self, lock_name): - setattr(self, lock_name, threading.Lock()) - lock = getattr(self, lock_name) - with lock: - return method(self, *args, **kws) - return sync_method - - -@synchronized_method -async def write_meta(request): - async with aiofiles.open("data/meta.txt", "r") as fs: - num = int(await fs.read()) + 1 - async with aiofiles.open("data/meta.txt", "w") as fs: - await fs.write(str(num)) - return num diff --git a/FileStorage/views/__init__.py b/FileStorage/views/__init__.py deleted file mode 100644 index 2ecea32..0000000 --- a/FileStorage/views/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .get_file import get_file -from .upload_file import upload_file -from .delete_file import delete_file diff --git a/FileStorage/views/delete_file.py b/FileStorage/views/delete_file.py deleted file mode 100644 index c581cf6..0000000 --- a/FileStorage/views/delete_file.py +++ /dev/null @@ -1,8 +0,0 @@ -from os import remove - -from aiohttp import web - - -async def delete_file(request): - remove("data/" + request.rel_url.query['id']) - return web.json_response({"success": True}) diff --git a/FileStorage/views/get_file.py b/FileStorage/views/get_file.py deleted file mode 100644 index e89cc2a..0000000 --- a/FileStorage/views/get_file.py +++ /dev/null @@ -1,10 +0,0 @@ -import aiofiles -from aiohttp import web - - -async def get_file(request): - response = web.StreamResponse() - await response.prepare(request) - async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs: - await response.write_eof(await fs.read()) - return response diff --git a/FileStorage/views/upload_file.py b/FileStorage/views/upload_file.py deleted file mode 100644 index d0a2593..0000000 --- a/FileStorage/views/upload_file.py +++ /dev/null @@ -1,14 +0,0 @@ -from aiohttp import web - -from FileStorage.sync import write_meta -import aiofiles - -from SprintLib.redis import lock - - -@lock() -async def upload_file(request): - file_id = await write_meta(request) - async with aiofiles.open("data/" + str(file_id), "wb") as fs: - await fs.write(await request.content.read()) - return web.json_response({"id": file_id}) diff --git a/Sprint/settings.py b/Sprint/settings.py index 3e5a81d..3829e2e 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -145,8 +145,9 @@ RABBIT_HOST = os.getenv("RABBIT_HOST", "127.0.0.1") RABBIT_PORT = 5672 RABBIT_PASSWORD = os.getenv("RABBIT_PASSWORD", "guest") -FS_HOST = "http://" + os.getenv("FS_HOST", "127.0.0.1") -FS_PORT = 5555 +MINIO_HOST = os.getenv("MINIO_HOST", "localhost") + ":9000" +MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "serviceminioadmin") +MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin") EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" EMAIL_HOST = "smtp.yandex.ru" diff --git a/SprintLib/utils.py b/SprintLib/utils.py index f8d4f9c..be4343e 100644 --- a/SprintLib/utils.py +++ b/SprintLib/utils.py @@ -1,38 +1,48 @@ import datetime +import io from random import choice from time import sleep from django.core.management import BaseCommand -from requests import get, post +from minio import Minio from Sprint import settings from SprintLib.queue import send_to_queue +from SprintLib.redis import lock + +BUCKET_NAME = 'dev' +client = Minio( + settings.MINIO_HOST, + access_key=settings.MINIO_ACCESS_KEY, + secret_key=settings.MINIO_SECRET_KEY, + secure=False +) + + +@lock('write_bytes') def write_bytes(data: bytes): - url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file" - print(url) - try: - return post(url, data=data).json()["id"] - except Exception: - return 0 + obj = client.get_object(BUCKET_NAME, 'meta.txt') + num = int(obj.data.decode('utf-8')) + 1 + b_num = str(num).encode('utf-8') + client.put_object(BUCKET_NAME, str(num), io.BytesIO(data), len(data)) + client.put_object(BUCKET_NAME, 'meta.txt', io.BytesIO(b_num), len(b_num)) + return num def get_bytes(num: int) -> bytes: - url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num) - print(url) try: - return get(url).content - except Exception: + return client.get_object(BUCKET_NAME, str(num)).data + except Exception as e: + print(e.with_traceback(None)) return b'' def delete_file(num: int): - url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num) - print(url) try: - post(url) - except Exception: + client.remove_object(BUCKET_NAME, str(num)) + except: ... diff --git a/requirements.txt b/requirements.txt index 57239c9..8245000 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ gunicorn==20.1.0 idna==3.2 importlib-metadata==4.5.0 kombu==5.1.0 +minio==7.1.11 pika==1.2.0 Pillow==8.3.1 prompt-toolkit==3.0.18