From edb58e23a32480bc1c38cffab95ee5dd18817476 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 11:47:06 +0300 Subject: [PATCH] filestorage --- .gitlab-ci.yml | 1 + FileStorage/__init__.py | 0 FileStorage/root.py | 21 +++++++++++ FileStorage/routes.py | 9 +++++ FileStorage/sync.py | 26 +++++++++++++ FileStorage/views/__init__.py | 3 ++ FileStorage/views/delete_file.py | 8 ++++ FileStorage/views/get_file.py | 10 +++++ FileStorage/views/upload_file.py | 11 ++++++ Main/management/commands/storage.py | 9 +++++ Main/models/__init__.py | 1 + Main/models/extrafile.py | 21 +++-------- Main/models/mixins.py | 14 +++++++ Main/models/solution.py | 49 +++++++++++------------- Main/models/solution_file.py | 9 +++++ Main/views/TaskSettingsView.py | 13 +++---- Main/views/TaskView.py | 58 ++++++++++++++++++----------- Sprint/settings.py | 3 ++ SprintLib/testers/BaseTester.py | 23 +++++++----- SprintLib/utils.py | 29 ++++++++++----- docker-compose.yaml | 10 +++++ requirements.txt | 2 + 22 files changed, 237 insertions(+), 93 deletions(-) create mode 100644 FileStorage/__init__.py create mode 100644 FileStorage/root.py create mode 100644 FileStorage/routes.py create mode 100644 FileStorage/sync.py create mode 100644 FileStorage/views/__init__.py create mode 100644 FileStorage/views/delete_file.py create mode 100644 FileStorage/views/get_file.py create mode 100644 FileStorage/views/upload_file.py create mode 100644 Main/management/commands/storage.py create mode 100644 Main/models/mixins.py create mode 100644 Main/models/solution_file.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f990870..391b7bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,7 @@ deploy-dev: SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" RABBIT_HOST: "rabbitmq" + FS_HOST: "storage" deploy-prod: extends: diff --git a/FileStorage/__init__.py b/FileStorage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FileStorage/root.py b/FileStorage/root.py new file mode 100644 index 0000000..3541645 --- /dev/null +++ b/FileStorage/root.py @@ -0,0 +1,21 @@ +import os +from os import mkdir +from os.path import exists + +from aiohttp import web +from FileStorage.routes import setup_routes + + +def runserver(): + app = web.Application() + setup_routes(app) + if not exists("data"): + mkdir("data") + if not exists("data/meta.txt"): + with open("data/meta.txt", "w") as fs: + fs.write("0") + web.run_app(app, host=os.getenv("FS_HOST", "0.0.0.0"), port=5555) + + +if __name__ == "__main__": + runserver() diff --git a/FileStorage/routes.py b/FileStorage/routes.py new file mode 100644 index 0000000..3ce9fcf --- /dev/null +++ b/FileStorage/routes.py @@ -0,0 +1,9 @@ +from aiohttp import web + +from FileStorage.views import get_file, upload_file, delete_file + + +def setup_routes(app: web.Application): + app.router.add_get("/get_file", get_file) + app.router.add_post("/upload_file", upload_file) + app.router.add_post("/delete_file", delete_file) diff --git a/FileStorage/sync.py b/FileStorage/sync.py new file mode 100644 index 0000000..fd82ce3 --- /dev/null +++ b/FileStorage/sync.py @@ -0,0 +1,26 @@ +import threading + +import aiofiles + + +def synchronized_method(method): + outer_lock = threading.Lock() + lock_name = "__" + method.__name__ + "_lock" + "__" + + def sync_method(self, *args, **kws): + with outer_lock: + if not hasattr(self, lock_name): + setattr(self, lock_name, threading.Lock()) + lock = getattr(self, lock_name) + with lock: + return method(self, *args, **kws) + return sync_method + + +@synchronized_method +async def write_meta(request): + async with aiofiles.open("data/meta.txt", "r") as fs: + num = int(await fs.read()) + 1 + async with aiofiles.open("data/meta.txt", "w") as fs: + await fs.write(str(num)) + return num diff --git a/FileStorage/views/__init__.py b/FileStorage/views/__init__.py new file mode 100644 index 0000000..2ecea32 --- /dev/null +++ b/FileStorage/views/__init__.py @@ -0,0 +1,3 @@ +from .get_file import get_file +from .upload_file import upload_file +from .delete_file import delete_file diff --git a/FileStorage/views/delete_file.py b/FileStorage/views/delete_file.py new file mode 100644 index 0000000..c581cf6 --- /dev/null +++ b/FileStorage/views/delete_file.py @@ -0,0 +1,8 @@ +from os import remove + +from aiohttp import web + + +async def delete_file(request): + remove("data/" + request.rel_url.query['id']) + return web.json_response({"success": True}) diff --git a/FileStorage/views/get_file.py b/FileStorage/views/get_file.py new file mode 100644 index 0000000..e89cc2a --- /dev/null +++ b/FileStorage/views/get_file.py @@ -0,0 +1,10 @@ +import aiofiles +from aiohttp import web + + +async def get_file(request): + response = web.StreamResponse() + await response.prepare(request) + async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs: + await response.write_eof(await fs.read()) + return response diff --git a/FileStorage/views/upload_file.py b/FileStorage/views/upload_file.py new file mode 100644 index 0000000..c5f2ff6 --- /dev/null +++ b/FileStorage/views/upload_file.py @@ -0,0 +1,11 @@ +from aiohttp import web + +from FileStorage.sync import write_meta +import aiofiles + + +async def upload_file(request): + file_id = await write_meta(request) + async with aiofiles.open("data/" + str(file_id), "wb") as fs: + await fs.write(await request.content.read()) + return web.json_response({"id": file_id}) diff --git a/Main/management/commands/storage.py b/Main/management/commands/storage.py new file mode 100644 index 0000000..1263566 --- /dev/null +++ b/Main/management/commands/storage.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from FileStorage.root import runserver + + +class Command(BaseCommand): + help = 'starts FileStorage' + + def handle(self, *args, **options): + runserver() diff --git a/Main/models/__init__.py b/Main/models/__init__.py index 393b604..36d607c 100644 --- a/Main/models/__init__.py +++ b/Main/models/__init__.py @@ -8,3 +8,4 @@ from Main.models.settask import SetTask from Main.models.solution import Solution from Main.models.extrafile import ExtraFile from Main.models.progress import Progress +from Main.models.solution_file import SolutionFile diff --git a/Main/models/extrafile.py b/Main/models/extrafile.py index bb7c511..c2895a3 100644 --- a/Main/models/extrafile.py +++ b/Main/models/extrafile.py @@ -1,23 +1,17 @@ -from os import remove -from os.path import join, exists - from django.core.exceptions import ObjectDoesNotExist from django.db import models -from Sprint.settings import DATA_ROOT +from .mixins import FileStorageMixin -class ExtraFile(models.Model): +class ExtraFile(FileStorageMixin, models.Model): task = models.ForeignKey("Task", on_delete=models.CASCADE) filename = models.TextField() is_test = models.BooleanField(null=True) is_sample = models.BooleanField(null=True) readable = models.BooleanField(null=True) test_number = models.IntegerField(null=True) - - @property - def path(self): - return join(DATA_ROOT, "extra_files", str(self.id)) + fs_id = models.IntegerField(null=True) @property def can_be_sample(self): @@ -29,13 +23,8 @@ class ExtraFile(models.Model): ) ) - @property - def text(self): - return open(self.path, "r").read() - def delete(self, using=None, keep_parents=False): - if exists(self.path): - remove(self.path) + self.remove_from_fs() if self.is_test and self.filename.endswith('.a'): try: ef = ExtraFile.objects.get(task=self.task, filename=self.filename.rstrip('.a'), is_test=True) @@ -47,4 +36,4 @@ class ExtraFile(models.Model): @property def answer(self): - return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a') \ No newline at end of file + return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a') diff --git a/Main/models/mixins.py b/Main/models/mixins.py new file mode 100644 index 0000000..1b3aba7 --- /dev/null +++ b/Main/models/mixins.py @@ -0,0 +1,14 @@ +from SprintLib.utils import get_bytes, write_bytes, delete_file + + +class FileStorageMixin: + @property + def text(self): + return get_bytes(self.fs_id).decode("utf-8") + + def write(self, bytes): + self.fs_id = write_bytes(bytes) + self.save() + + def remove_from_fs(self): + delete_file(self.fs_id) diff --git a/Main/models/solution.py b/Main/models/solution.py index 52256ad..9e365dd 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -1,13 +1,12 @@ -from os import mkdir, walk from os.path import join, exists from shutil import rmtree from subprocess import call from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.utils import timezone +from Main.models.solution_file import SolutionFile from Main.models.task import Task from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL from SprintLib.language import languages @@ -32,36 +31,30 @@ class Solution(models.Model): @property def files(self): data = [] - for path, _, files in walk(self.directory): - if path.startswith(self.testing_directory): + for file in SolutionFile.objects.filter(solution=self): + try: + text = file.text + except: continue - for file in files: - try: - entity = { - 'filename': file, - 'text': open(join(path, file), 'r').read() - } - end = file.split('.')[-1] - language = None - for l in languages: - if l.file_type == end: - language = l - break - if language is None: - highlight = 'nohighlight' - else: - highlight = 'language-' + language.highlight - entity['highlight'] = highlight - data.append(entity) - except: - continue + entity = { + 'filename': file.path, + 'text': text + } + end = file.path.split('.')[-1] + language = None + for l in languages: + if l.file_type == end: + language = l + break + if language is None: + highlight = 'nohighlight' + else: + highlight = 'language-' + language.highlight + entity['highlight'] = highlight + data.append(entity) data.sort(key=lambda x: x['filename']) return data - def create_dirs(self): - mkdir(self.directory) - mkdir(self.testing_directory) - @property def directory(self): return join(SOLUTIONS_ROOT, str(self.id)) diff --git a/Main/models/solution_file.py b/Main/models/solution_file.py new file mode 100644 index 0000000..702b5ff --- /dev/null +++ b/Main/models/solution_file.py @@ -0,0 +1,9 @@ +from django.db import models + +from Main.models.mixins import FileStorageMixin + + +class SolutionFile(FileStorageMixin, models.Model): + path = models.TextField() + fs_id = models.IntegerField() + solution = models.ForeignKey('Solution', on_delete=models.CASCADE) diff --git a/Main/views/TaskSettingsView.py b/Main/views/TaskSettingsView.py index e0998c6..c32a167 100644 --- a/Main/views/TaskSettingsView.py +++ b/Main/views/TaskSettingsView.py @@ -45,11 +45,9 @@ class TaskSettingsView(BaseView): filename=filename, is_test=is_test ) - with open(ef.path, 'wb') as fs: - for chunk in self.request.FILES['file'].chunks(): - fs.write(chunk) + ef.write(self.request.FILES['file'].read()) try: - open(ef.path, 'r').read() + var = ef.text ef.readable = True except UnicodeDecodeError: ef.readable = False @@ -73,8 +71,7 @@ class TaskSettingsView(BaseView): ef, created = ExtraFile.objects.get_or_create(filename=name, task=self.entities.task) if not created: return f'/admin/task?task_id={self.entities.task.id}&error_message=Файл с таким именем уже существует' - with open(ef.path, 'w') as fs: - fs.write('') + ef.write(b"") ef.is_test = is_test ef.readable = True ef.save() @@ -88,8 +85,8 @@ class TaskSettingsView(BaseView): def post_save_test(self): ef = ExtraFile.objects.get(id=self.request.POST['test_id']) - with open(ef.path, 'w') as fs: - fs.write(self.request.POST['text']) + ef.remove_from_fs() + ef.write(self.request.POST['text'].encode('utf-8')) ef.is_sample = 'is_sample' in self.request.POST.keys() ef.save() return f'/admin/task?task_id={self.entities.task.id}' diff --git a/Main/views/TaskView.py b/Main/views/TaskView.py index 5859fea..aeae5e3 100644 --- a/Main/views/TaskView.py +++ b/Main/views/TaskView.py @@ -1,11 +1,11 @@ +import io from zipfile import ZipFile -from os.path import join -from Main.models import Solution, Progress +from Main.models import Solution, Progress, SolutionFile from SprintLib.BaseView import BaseView from SprintLib.language import languages from SprintLib.queue import send_testing -from SprintLib.testers import * +from SprintLib.utils import write_bytes class TaskView(BaseView): @@ -13,40 +13,54 @@ class TaskView(BaseView): view_file = "task.html" def get(self): - self.context['languages'] = sorted(languages, key=lambda x: x.name) - progress, _ = Progress.objects.get_or_create(user=self.request.user, task=self.entities.task) - self.context['progress'] = progress + self.context["languages"] = sorted(languages, key=lambda x: x.name) + progress, _ = Progress.objects.get_or_create( + user=self.request.user, task=self.entities.task + ) + self.context["progress"] = progress def pre_handle(self): - if self.request.method == 'GET': + if self.request.method == "GET": return self.solution = Solution.objects.create( task=self.entities.task, user=self.request.user, - language_id=int(self.request.POST["language"]) + language_id=int(self.request.POST["language"]), ) - self.solution.create_dirs() def post_0(self): # отправка решения через текст - filename = 'solution.' + self.solution.language.file_type - file_path = join(self.solution.directory, filename) - with open(file_path, 'w') as fs: - fs.write(self.request.POST['code']) + fs_id = write_bytes(self.request.POST["code"].encode("utf-8")) + SolutionFile.objects.create( + path="solution." + self.solution.language.file_type, + solution=self.solution, + fs_id=fs_id, + ) send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) def post_1(self): # отправка решения через файл - if 'file' not in self.request.FILES: + if "file" not in self.request.FILES: return "task?task_id=" + str(self.entities.task.id) - filename = self.request.FILES['file'].name - file_path = join(self.solution.directory, filename) - with open(file_path, 'wb') as fs: - for chunk in self.request.FILES['file'].chunks(): - fs.write(chunk) - if filename.endswith('.zip'): - with ZipFile(file_path) as obj: - obj.extractall(self.solution.directory) + filename = self.request.FILES["file"].name + if filename.endswith(".zip"): + archive = ZipFile(io.BytesIO(self.request.FILES['file'].read())) + for file in archive.infolist(): + if file.is_dir(): + continue + fs_id = write_bytes(archive.read(file.filename)) + SolutionFile.objects.create( + path=file.filename, + solution=self.solution, + fs_id=fs_id, + ) + else: + fs_id = write_bytes(self.request.FILES['file'].read()) + SolutionFile.objects.create( + path=filename, + solution=self.solution, + fs_id=fs_id, + ) send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) diff --git a/Sprint/settings.py b/Sprint/settings.py index 4ea8baa..18847a2 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -143,6 +143,9 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") RABBIT_HOST = os.getenv("RABBIT_HOST", "0.0.0.0") RABBIT_PORT = 5672 +FS_HOST = os.getenv("FS_HOST", "http://0.0.0.0") +FS_PORT = 5555 + STATICFILES_DIRS = [ os.path.join(BASE_DIR, "Main/static"), ] diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 69b1cf5..7e763c7 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -4,10 +4,10 @@ from shutil import copyfile, rmtree from subprocess import call, TimeoutExpired from Main.management.commands.bot import bot -from Main.models import ExtraFile +from Main.models import ExtraFile, SolutionFile from Main.models.progress import Progress from Sprint.settings import CONSTS -from SprintLib.utils import copy_content +from SprintLib.utils import get_bytes class TestException(Exception): @@ -56,11 +56,15 @@ class BaseTester: self.solution = solution def execute(self): - if not exists(self.solution.testing_directory): - mkdir(self.solution.testing_directory) - copy_content( - self.solution.directory, self.solution.testing_directory, ("test_dir",) - ) + mkdir("solution") + for file in SolutionFile.objects.filter(solution=self.solution): + dirs = file.path.split('/') + for i in range(len(dirs) - 1): + name = join("solution", '/'.join(dirs[:i + 1])) + if not exists(name): + mkdir(name) + with open(file.path, 'wb') as fs: + fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" @@ -68,7 +72,8 @@ class BaseTester: call(docker_command, shell=True) print("Container created") for file in ExtraFile.objects.filter(task=self.solution.task): - copyfile(file.path, join(self.solution.testing_directory, file.filename)) + with open(join("solution", file.filename), 'wb') as fs: + fs.write(get_bytes(file.fs_id)) print("Files copied") try: self.before_test() @@ -98,7 +103,7 @@ class BaseTester: print(str(e)) self.solution.save() call(f"docker rm --force solution_{self.solution.id}", shell=True) - rmtree(self.solution.testing_directory) + rmtree("solution") self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: bot.send_message( diff --git a/SprintLib/utils.py b/SprintLib/utils.py index df682d3..751538f 100644 --- a/SprintLib/utils.py +++ b/SprintLib/utils.py @@ -1,12 +1,21 @@ -from os import listdir -from os.path import isfile, join -from shutil import copyfile, copytree +from requests import get, post + +from Sprint import settings -def copy_content(from_dir, to_dir, exc=()): - for file in listdir(from_dir): - if file in exc: - continue - full_path = join(from_dir, file) - func = copyfile if isfile(full_path) else copytree - func(full_path, join(to_dir, file)) +def write_bytes(data): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file" + print(url) + return post(url, data=data).json()['id'] + + +def get_bytes(num): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num) + print(url) + return get(url).content + + +def delete_file(num): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num) + print(url) + post(url) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0eaa7eb..1a65cce 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,6 +24,7 @@ services: PORT: $PORT DB_HOST: $DB_HOST RABBIT_HOST: $RABBIT_HOST + FS_HOST: $FS_HOST command: scripts/runserver.sh ports: - "${PORT}:${PORT}" @@ -33,6 +34,15 @@ services: depends_on: - postgres - rabbitmq + - storage + + storage: + restart: always + image: mathwave/sprint-repo:sprint + ports: + - "5555:5555" + volumes: + - /sprint-data/data:/usr/src/app/FileStorage/data bot: image: mathwave/sprint-repo:sprint diff --git a/requirements.txt b/requirements.txt index dbd0f50..d6566bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +aiofiles==0.7.0 +aiohttp==3.8.0 amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0