minio

.deploy/deploy-dev.yaml

@@ -27,9 +27,10 @@ services:
       - net
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV
@@ -51,9 +52,10 @@ services:
       - net
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV
@@ -68,42 +70,16 @@ services:
         parallelism: 1
         order: start-first
 
-  storage:
-    image: mathwave/sprint-repo:sprint
-    networks:
-      - net
-    command: ./manage.py storage
-    environment:
-      DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
-      RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
-      REDIS_HOST: "redis.develop.sprinthub.ru"
-      RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
-      REDIS_PASSWORD: $REDIS_PASSWORD_DEV
-      DB_PASSWORD: $DB_PASSWORD_DEV
-      DEBUG: "true"
-      TELEGRAM_TOKEN: $TELEGRAM_TOKEN_DEV
-    volumes:
-      - /sprint-data/data:/usr/src/app/data
-    deploy:
-      mode: replicated
-      restart_policy:
-        condition: any
-      placement:
-        constraints: [node.role == manager]
-      update_config:
-        parallelism: 1
-        order: start-first
-
   bot:
     image: mathwave/sprint-repo:sprint
     networks:
       - net
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV
@@ -124,9 +100,10 @@ services:
       - net
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV
@@ -148,9 +125,10 @@ services:
     command: ./manage.py receive
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV
@@ -175,9 +153,10 @@ services:
     command: ./manage.py file_generator
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV
@@ -199,9 +178,10 @@ services:
     command: ./manage.py notification_manager
     environment:
       DB_HOST: "pg.develop.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
       REDIS_HOST: "redis.develop.sprinthub.ru"
+      MINIO_HOST: "minio.develop.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
       REDIS_PASSWORD: $REDIS_PASSWORD_DEV
       DB_PASSWORD: $DB_PASSWORD_DEV

.deploy/deploy-prod.yaml

@@ -21,43 +21,15 @@ services:
         parallelism: 1
         order: start-first
 
-  storage:
-    image: mathwave/sprint-repo:sprint
-    networks:
-      - net
-    command: ./manage.py storage
-    environment:
-      DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
-      RABBIT_HOST: "rabbitmq.sprinthub.ru"
-      REDIS_HOST: "redis.sprinthub.ru"
-      RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
-      REDIS_PASSWORD: $REDIS_PASSWORD_PROD
-      DB_PASSWORD: $DB_PASSWORD_PROD
-      DEBUG: "false"
-      TELEGRAM_TOKEN: $TELEGRAM_TOKEN_PROD
-      SENTRY_TOKEN: $SENTRY_TOKEN
-    volumes:
-      - /sprint-data/data:/usr/src/app/data
-    deploy:
-      mode: replicated
-      restart_policy:
-        condition: any
-      placement:
-        constraints: [node.role == manager]
-      update_config:
-        parallelism: 1
-        order: start-first
-
   web:
     image: mathwave/sprint-repo:sprint
     networks:
       - net
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD
@@ -79,9 +51,9 @@ services:
       - net
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD
@@ -104,9 +76,9 @@ services:
       - net
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD
@@ -129,9 +101,9 @@ services:
       - net
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD
@@ -154,9 +126,10 @@ services:
     command: ./manage.py receive
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD
@@ -182,9 +155,10 @@ services:
     command: ./manage.py file_generator
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD
@@ -207,9 +181,10 @@ services:
     command: ./manage.py notification_manager
     environment:
       DB_HOST: "pg.sprinthub.ru"
-      FS_HOST: "storage"
       RABBIT_HOST: "rabbitmq.sprinthub.ru"
       REDIS_HOST: "redis.sprinthub.ru"
+      MINIO_HOST: "minio.sprinthub.ru"
+      MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD
       RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
       REDIS_PASSWORD: $REDIS_PASSWORD_PROD
       DB_PASSWORD: $DB_PASSWORD_PROD

Checker/views.py

@@ -8,9 +8,9 @@ from django.http import JsonResponse, HttpResponse
 from django.utils import timezone
 
 from Checker.models import Checker
-from FileStorage.sync import synchronized_method
 from Main.models import Solution, SolutionFile, ExtraFile, Progress
 from SprintLib.queue import send_to_queue
+from SprintLib.redis import lock
 from SprintLib.utils import generate_token
 
 
@@ -37,7 +37,7 @@ def status(request):
         return JsonResponse({"status": "incorrect token"}, status=403)
 
 
-@synchronized_method
+@lock('checker')
 def available(request):
     try:
         checker = Checker.objects.get(dynamic_token=request.GET['token'])

FileStorage/root.py

@@ -1,21 +0,0 @@
-from os import mkdir
-from os.path import exists
-
-from aiohttp import web
-
-from FileStorage.routes import setup_routes
-
-
-def runserver():
-    app = web.Application()
-    setup_routes(app)
-    if not exists("data"):
-        mkdir("data")
-    if not exists("data/meta.txt"):
-        with open("data/meta.txt", "w") as fs:
-            fs.write("0")
-    web.run_app(app, host="0.0.0.0", port=5555)
-
-
-if __name__ == "__main__":
-    runserver()

FileStorage/routes.py

@@ -1,9 +0,0 @@
-from aiohttp import web
-
-from FileStorage.views import get_file, upload_file, delete_file
-
-
-def setup_routes(app: web.Application):
-    app.router.add_get("/get_file", get_file)
-    app.router.add_post("/upload_file", upload_file)
-    app.router.add_post("/delete_file", delete_file)

FileStorage/sync.py

@@ -1,26 +0,0 @@
-import threading
-
-import aiofiles
-
-
-def synchronized_method(method):
-    outer_lock = threading.Lock()
-    lock_name = "__" + method.__name__ + "_lock" + "__"
-
-    def sync_method(self, *args, **kws):
-        with outer_lock:
-            if not hasattr(self, lock_name):
-                setattr(self, lock_name, threading.Lock())
-            lock = getattr(self, lock_name)
-            with lock:
-                return method(self, *args, **kws)
-    return sync_method
-
-
-@synchronized_method
-async def write_meta(request):
-    async with aiofiles.open("data/meta.txt", "r") as fs:
-        num = int(await fs.read()) + 1
-    async with aiofiles.open("data/meta.txt", "w") as fs:
-        await fs.write(str(num))
-    return num

FileStorage/views/__init__.py

@@ -1,3 +0,0 @@
-from .get_file import get_file
-from .upload_file import upload_file
-from .delete_file import delete_file

FileStorage/views/delete_file.py

@@ -1,8 +0,0 @@
-from os import remove
-
-from aiohttp import web
-
-
-async def delete_file(request):
-    remove("data/" + request.rel_url.query['id'])
-    return web.json_response({"success": True})

FileStorage/views/get_file.py

@@ -1,10 +0,0 @@
-import aiofiles
-from aiohttp import web
-
-
-async def get_file(request):
-    response = web.StreamResponse()
-    await response.prepare(request)
-    async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs:
-        await response.write_eof(await fs.read())
-    return response

FileStorage/views/upload_file.py

@@ -1,14 +0,0 @@
-from aiohttp import web
-
-from FileStorage.sync import write_meta
-import aiofiles
-
-from SprintLib.redis import lock
-
-
-@lock()
-async def upload_file(request):
-    file_id = await write_meta(request)
-    async with aiofiles.open("data/" + str(file_id), "wb") as fs:
-        await fs.write(await request.content.read())
-    return web.json_response({"id": file_id})

Sprint/settings.py

@@ -145,8 +145,9 @@ RABBIT_HOST = os.getenv("RABBIT_HOST", "127.0.0.1")
 RABBIT_PORT = 5672
 RABBIT_PASSWORD = os.getenv("RABBIT_PASSWORD", "guest")
 
-FS_HOST = "http://" + os.getenv("FS_HOST", "127.0.0.1")
+MINIO_HOST = os.getenv("MINIO_HOST", "localhost") + ":9000"
-FS_PORT = 5555
+MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "serviceminioadin")
+MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin")
 
 EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
 EMAIL_HOST = "smtp.yandex.ru"

SprintLib/utils.py

@@ -1,38 +1,47 @@
 import datetime
+import io
 from random import choice
 from time import sleep
 
 from django.core.management import BaseCommand
-from requests import get, post
+from minio import Minio
 
 from Sprint import settings
 from SprintLib.queue import send_to_queue
+from SprintLib.redis import lock
+
+BUCKET_NAME = 'dev'
 
 
+client = Minio(
+    settings.MINIO_HOST,
+    access_key=settings.MINIO_ACCESS_KEY,
+    secret_key=settings.MINIO_SECRET_KEY,
+    secure=False
+)
+
+
+@lock('write_bytes')
 def write_bytes(data: bytes):
-    url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file"
+    obj = client.get_object(BUCKET_NAME, 'meta.txt')
-    print(url)
+    num = int(obj.data.decode('utf-8')) + 1
-    try:
+    b_num = str(num).encode('utf-8')
-        return post(url, data=data).json()["id"]
+    client.put_object(BUCKET_NAME, str(num), io.BytesIO(data), len(data))
-    except Exception:
+    client.put_object(BUCKET_NAME, 'meta.txt', io.BytesIO(b_num), len(b_num))
-        return 0
+    return num
 
 
 def get_bytes(num: int) -> bytes:
-    url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num)
-    print(url)
     try:
-        return get(url).content
+        return client.get_object(BUCKET_NAME, str(num)).data
-    except Exception:
+    except:
         return b''
 
 
 def delete_file(num: int):
-    url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num)
-    print(url)
     try:
-        post(url)
+        client.remove_object(BUCKET_NAME, str(num))
-    except Exception:
+    except:
         ...
 
 

requirements.txt

@@ -16,6 +16,7 @@ gunicorn==20.1.0
 idna==3.2
 importlib-metadata==4.5.0
 kombu==5.1.0
+minio==7.1.11
 pika==1.2.0
 Pillow==8.3.1
 prompt-toolkit==3.0.18