self/sprint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'minio' into 'master'

Browse Source 
Minio

See merge request self/Sprint!1
This commit is contained in:
Egor Matveev 2022-08-21 12:40:14 +00:00
commit ab1aee1c87
14 changed files with 55 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.deploy/deploy-dev.yaml
View File

@ -27,9 +27,10 @@ services:
- net
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
@ -51,9 +52,10 @@ services:
- net
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
@ -68,42 +70,16 @@ services:
parallelism: 1
order: start-first
storage:
image: mathwave/sprint-repo:sprint
networks:
- net
command: ./manage.py storage
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
DEBUG: "true"
TELEGRAM_TOKEN: $TELEGRAM_TOKEN_DEV
volumes:
- /sprint-data/data:/usr/src/app/data
deploy:
mode: replicated
restart_policy:
condition: any
placement:
constraints: [node.role == manager]
update_config:
parallelism: 1
order: start-first
bot:
image: mathwave/sprint-repo:sprint
networks:
- net
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
@ -124,9 +100,10 @@ services:
- net
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
@ -148,9 +125,10 @@ services:
command: ./manage.py receive
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
@ -175,9 +153,10 @@ services:
command: ./manage.py file_generator
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV
@ -199,9 +178,10 @@ services:
command: ./manage.py notification_manager
environment:
DB_HOST: "pg.develop.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.develop.sprinthub.ru"
REDIS_HOST: "redis.develop.sprinthub.ru"
MINIO_HOST: "minio.develop.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_DEV
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_DEV
REDIS_PASSWORD: $REDIS_PASSWORD_DEV
DB_PASSWORD: $DB_PASSWORD_DEV

45
.deploy/deploy-prod.yaml
View File

@ -21,43 +21,15 @@ services:
parallelism: 1
order: start-first
storage:
image: mathwave/sprint-repo:sprint
networks:
- net
command: ./manage.py storage
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
DEBUG: "false"
TELEGRAM_TOKEN: $TELEGRAM_TOKEN_PROD
SENTRY_TOKEN: $SENTRY_TOKEN
volumes:
- /sprint-data/data:/usr/src/app/data
deploy:
mode: replicated
restart_policy:
condition: any
placement:
constraints: [node.role == manager]
update_config:
parallelism: 1
order: start-first
web:
image: mathwave/sprint-repo:sprint
networks:
- net
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
@ -79,9 +51,9 @@ services:
- net
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
@ -104,9 +76,9 @@ services:
- net
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
@ -129,9 +101,9 @@ services:
- net
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
@ -154,9 +126,10 @@ services:
command: ./manage.py receive
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
@ -182,9 +155,10 @@ services:
command: ./manage.py file_generator
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD
@ -207,9 +181,10 @@ services:
command: ./manage.py notification_manager
environment:
DB_HOST: "pg.sprinthub.ru"
FS_HOST: "storage"
RABBIT_HOST: "rabbitmq.sprinthub.ru"
REDIS_HOST: "redis.sprinthub.ru"
MINIO_HOST: "minio.sprinthub.ru"
MINIO_SECRET_KEY: $MINIO_SECRET_KEY_PROD
RABBIT_PASSWORD: $RABBITMQ_PASSWORD_PROD
REDIS_PASSWORD: $REDIS_PASSWORD_PROD
DB_PASSWORD: $DB_PASSWORD_PROD

4
Checker/views.py
View File

@ -8,9 +8,9 @@ from django.http import JsonResponse, HttpResponse
from django.utils import timezone
from Checker.models import Checker
from FileStorage.sync import synchronized_method
from Main.models import Solution, SolutionFile, ExtraFile, Progress
from SprintLib.queue import send_to_queue
from SprintLib.redis import lock
from SprintLib.utils import generate_token
@ -37,7 +37,7 @@ def status(request):
return JsonResponse({"status": "incorrect token"}, status=403)
@synchronized_method
@lock('checker')
def available(request):
try:
checker = Checker.objects.get(dynamic_token=request.GET['token'])

0
FileStorage/__init__.py
View File

21
FileStorage/root.py
View File

@ -1,21 +0,0 @@
from os import mkdir
from os.path import exists
from aiohttp import web
from FileStorage.routes import setup_routes
def runserver():
app = web.Application()
setup_routes(app)
if not exists("data"):
mkdir("data")
if not exists("data/meta.txt"):
with open("data/meta.txt", "w") as fs:
fs.write("0")
web.run_app(app, host="0.0.0.0", port=5555)
if __name__ == "__main__":
runserver()

9
FileStorage/routes.py
View File

@ -1,9 +0,0 @@
from aiohttp import web
from FileStorage.views import get_file, upload_file, delete_file
def setup_routes(app: web.Application):
app.router.add_get("/get_file", get_file)
app.router.add_post("/upload_file", upload_file)
app.router.add_post("/delete_file", delete_file)

26
FileStorage/sync.py
View File

@ -1,26 +0,0 @@
import threading
import aiofiles
def synchronized_method(method):
outer_lock = threading.Lock()
lock_name = "__" + method.__name__ + "_lock" + "__"
def sync_method(self, *args, **kws):
with outer_lock:
if not hasattr(self, lock_name):
setattr(self, lock_name, threading.Lock())
lock = getattr(self, lock_name)
with lock:
return method(self, *args, **kws)
return sync_method
@synchronized_method
async def write_meta(request):
async with aiofiles.open("data/meta.txt", "r") as fs:
num = int(await fs.read()) + 1
async with aiofiles.open("data/meta.txt", "w") as fs:
await fs.write(str(num))
return num

3
FileStorage/views/__init__.py
View File

@ -1,3 +0,0 @@
from .get_file import get_file
from .upload_file import upload_file
from .delete_file import delete_file

8
FileStorage/views/delete_file.py
View File

@ -1,8 +0,0 @@
from os import remove
from aiohttp import web
async def delete_file(request):
remove("data/" + request.rel_url.query['id'])
return web.json_response({"success": True})

10
FileStorage/views/get_file.py
View File

@ -1,10 +0,0 @@
import aiofiles
from aiohttp import web
async def get_file(request):
response = web.StreamResponse()
await response.prepare(request)
async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs:
await response.write_eof(await fs.read())
return response

14
FileStorage/views/upload_file.py
View File

@ -1,14 +0,0 @@
from aiohttp import web
from FileStorage.sync import write_meta
import aiofiles
from SprintLib.redis import lock
@lock()
async def upload_file(request):
file_id = await write_meta(request)
async with aiofiles.open("data/" + str(file_id), "wb") as fs:
await fs.write(await request.content.read())
return web.json_response({"id": file_id})

5
Sprint/settings.py
View File

@ -145,8 +145,9 @@ RABBIT_HOST = os.getenv("RABBIT_HOST", "127.0.0.1")
RABBIT_PORT = 5672
RABBIT_PASSWORD = os.getenv("RABBIT_PASSWORD", "guest")
FS_HOST = "http://" + os.getenv("FS_HOST", "127.0.0.1")
FS_PORT = 5555
MINIO_HOST = os.getenv("MINIO_HOST", "localhost") + ":9000"
MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "serviceminioadmin")
MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin")
EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
EMAIL_HOST = "smtp.yandex.ru"

40
SprintLib/utils.py
View File

@ -1,38 +1,48 @@
import datetime
import io
from random import choice
from time import sleep
from django.core.management import BaseCommand
from requests import get, post
from minio import Minio
from Sprint import settings
from SprintLib.queue import send_to_queue
from SprintLib.redis import lock
BUCKET_NAME = 'dev'
client = Minio(
settings.MINIO_HOST,
access_key=settings.MINIO_ACCESS_KEY,
secret_key=settings.MINIO_SECRET_KEY,
secure=False
)
@lock('write_bytes')
def write_bytes(data: bytes):
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file"
print(url)
try:
return post(url, data=data).json()["id"]
except Exception:
return 0
obj = client.get_object(BUCKET_NAME, 'meta.txt')
num = int(obj.data.decode('utf-8')) + 1
b_num = str(num).encode('utf-8')
client.put_object(BUCKET_NAME, str(num), io.BytesIO(data), len(data))
client.put_object(BUCKET_NAME, 'meta.txt', io.BytesIO(b_num), len(b_num))
return num
def get_bytes(num: int) -> bytes:
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num)
print(url)
try:
return get(url).content
except Exception:
return client.get_object(BUCKET_NAME, str(num)).data
except Exception as e:
print(e.with_traceback(None))
return b''
def delete_file(num: int):
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num)
print(url)
try:
post(url)
except Exception:
client.remove_object(BUCKET_NAME, str(num))
except:
...

1
requirements.txt
View File

@ -16,6 +16,7 @@ gunicorn==20.1.0
idna==3.2
importlib-metadata==4.5.0
kombu==5.1.0
minio==7.1.11
pika==1.2.0
Pillow==8.3.1
prompt-toolkit==3.0.18