diff --git a/FileStorage/__init__.py b/FileStorage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FileStorage/root.py b/FileStorage/root.py new file mode 100644 index 0000000..3541645 --- /dev/null +++ b/FileStorage/root.py @@ -0,0 +1,21 @@ +import os +from os import mkdir +from os.path import exists + +from aiohttp import web +from FileStorage.routes import setup_routes + + +def runserver(): + app = web.Application() + setup_routes(app) + if not exists("data"): + mkdir("data") + if not exists("data/meta.txt"): + with open("data/meta.txt", "w") as fs: + fs.write("0") + web.run_app(app, host=os.getenv("FS_HOST", "0.0.0.0"), port=5555) + + +if __name__ == "__main__": + runserver() diff --git a/FileStorage/routes.py b/FileStorage/routes.py new file mode 100644 index 0000000..3ce9fcf --- /dev/null +++ b/FileStorage/routes.py @@ -0,0 +1,9 @@ +from aiohttp import web + +from FileStorage.views import get_file, upload_file, delete_file + + +def setup_routes(app: web.Application): + app.router.add_get("/get_file", get_file) + app.router.add_post("/upload_file", upload_file) + app.router.add_post("/delete_file", delete_file) diff --git a/FileStorage/sync.py b/FileStorage/sync.py new file mode 100644 index 0000000..fd82ce3 --- /dev/null +++ b/FileStorage/sync.py @@ -0,0 +1,26 @@ +import threading + +import aiofiles + + +def synchronized_method(method): + outer_lock = threading.Lock() + lock_name = "__" + method.__name__ + "_lock" + "__" + + def sync_method(self, *args, **kws): + with outer_lock: + if not hasattr(self, lock_name): + setattr(self, lock_name, threading.Lock()) + lock = getattr(self, lock_name) + with lock: + return method(self, *args, **kws) + return sync_method + + +@synchronized_method +async def write_meta(request): + async with aiofiles.open("data/meta.txt", "r") as fs: + num = int(await fs.read()) + 1 + async with aiofiles.open("data/meta.txt", "w") as fs: + await fs.write(str(num)) + return num diff --git a/FileStorage/views/__init__.py b/FileStorage/views/__init__.py new file mode 100644 index 0000000..2ecea32 --- /dev/null +++ b/FileStorage/views/__init__.py @@ -0,0 +1,3 @@ +from .get_file import get_file +from .upload_file import upload_file +from .delete_file import delete_file diff --git a/FileStorage/views/delete_file.py b/FileStorage/views/delete_file.py new file mode 100644 index 0000000..c581cf6 --- /dev/null +++ b/FileStorage/views/delete_file.py @@ -0,0 +1,8 @@ +from os import remove + +from aiohttp import web + + +async def delete_file(request): + remove("data/" + request.rel_url.query['id']) + return web.json_response({"success": True}) diff --git a/FileStorage/views/get_file.py b/FileStorage/views/get_file.py new file mode 100644 index 0000000..e89cc2a --- /dev/null +++ b/FileStorage/views/get_file.py @@ -0,0 +1,10 @@ +import aiofiles +from aiohttp import web + + +async def get_file(request): + response = web.StreamResponse() + await response.prepare(request) + async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs: + await response.write_eof(await fs.read()) + return response diff --git a/FileStorage/views/upload_file.py b/FileStorage/views/upload_file.py new file mode 100644 index 0000000..c5f2ff6 --- /dev/null +++ b/FileStorage/views/upload_file.py @@ -0,0 +1,11 @@ +from aiohttp import web + +from FileStorage.sync import write_meta +import aiofiles + + +async def upload_file(request): + file_id = await write_meta(request) + async with aiofiles.open("data/" + str(file_id), "wb") as fs: + await fs.write(await request.content.read()) + return web.json_response({"id": file_id}) diff --git a/Main/management/commands/storage.py b/Main/management/commands/storage.py new file mode 100644 index 0000000..1263566 --- /dev/null +++ b/Main/management/commands/storage.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from FileStorage.root import runserver + + +class Command(BaseCommand): + help = 'starts FileStorage' + + def handle(self, *args, **options): + runserver() diff --git a/Main/migrations/0075_auto_20211110_2317.py b/Main/migrations/0075_auto_20211110_2317.py new file mode 100644 index 0000000..0c4d95d --- /dev/null +++ b/Main/migrations/0075_auto_20211110_2317.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.4 on 2021-11-10 20:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('Main', '0074_auto_20211106_1215'), + ] + + operations = [ + migrations.CreateModel( + name='SolutionFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.TextField()), + ('fs_id', models.IntegerField()), + ('solution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='Main.solution')), + ], + ), + migrations.DeleteModel( + name='Language', + ), + migrations.AddField( + model_name='extrafile', + name='fs_id', + field=models.IntegerField(null=True), + ), + ] diff --git a/Main/models/__init__.py b/Main/models/__init__.py index 393b604..36d607c 100644 --- a/Main/models/__init__.py +++ b/Main/models/__init__.py @@ -8,3 +8,4 @@ from Main.models.settask import SetTask from Main.models.solution import Solution from Main.models.extrafile import ExtraFile from Main.models.progress import Progress +from Main.models.solution_file import SolutionFile diff --git a/Main/models/extrafile.py b/Main/models/extrafile.py index bb7c511..c2895a3 100644 --- a/Main/models/extrafile.py +++ b/Main/models/extrafile.py @@ -1,23 +1,17 @@ -from os import remove -from os.path import join, exists - from django.core.exceptions import ObjectDoesNotExist from django.db import models -from Sprint.settings import DATA_ROOT +from .mixins import FileStorageMixin -class ExtraFile(models.Model): +class ExtraFile(FileStorageMixin, models.Model): task = models.ForeignKey("Task", on_delete=models.CASCADE) filename = models.TextField() is_test = models.BooleanField(null=True) is_sample = models.BooleanField(null=True) readable = models.BooleanField(null=True) test_number = models.IntegerField(null=True) - - @property - def path(self): - return join(DATA_ROOT, "extra_files", str(self.id)) + fs_id = models.IntegerField(null=True) @property def can_be_sample(self): @@ -29,13 +23,8 @@ class ExtraFile(models.Model): ) ) - @property - def text(self): - return open(self.path, "r").read() - def delete(self, using=None, keep_parents=False): - if exists(self.path): - remove(self.path) + self.remove_from_fs() if self.is_test and self.filename.endswith('.a'): try: ef = ExtraFile.objects.get(task=self.task, filename=self.filename.rstrip('.a'), is_test=True) @@ -47,4 +36,4 @@ class ExtraFile(models.Model): @property def answer(self): - return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a') \ No newline at end of file + return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a') diff --git a/Main/models/mixins.py b/Main/models/mixins.py new file mode 100644 index 0000000..1b3aba7 --- /dev/null +++ b/Main/models/mixins.py @@ -0,0 +1,14 @@ +from SprintLib.utils import get_bytes, write_bytes, delete_file + + +class FileStorageMixin: + @property + def text(self): + return get_bytes(self.fs_id).decode("utf-8") + + def write(self, bytes): + self.fs_id = write_bytes(bytes) + self.save() + + def remove_from_fs(self): + delete_file(self.fs_id) diff --git a/Main/models/solution.py b/Main/models/solution.py index 52256ad..8c1a491 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -1,13 +1,12 @@ -from os import mkdir, walk from os.path import join, exists from shutil import rmtree from subprocess import call from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.utils import timezone +from Main.models.solution_file import SolutionFile from Main.models.task import Task from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL from SprintLib.language import languages @@ -32,47 +31,41 @@ class Solution(models.Model): @property def files(self): data = [] - for path, _, files in walk(self.directory): - if path.startswith(self.testing_directory): + for file in SolutionFile.objects.filter(solution=self): + try: + text = file.text + except: continue - for file in files: - try: - entity = { - 'filename': file, - 'text': open(join(path, file), 'r').read() - } - end = file.split('.')[-1] - language = None - for l in languages: - if l.file_type == end: - language = l - break - if language is None: - highlight = 'nohighlight' - else: - highlight = 'language-' + language.highlight - entity['highlight'] = highlight - data.append(entity) - except: - continue + entity = { + 'filename': file.path, + 'text': text + } + end = file.path.split('.')[-1] + language = None + for l in languages: + if l.file_type == end: + language = l + break + if language is None: + highlight = 'nohighlight' + else: + highlight = 'language-' + language.highlight + entity['highlight'] = highlight + data.append(entity) data.sort(key=lambda x: x['filename']) return data - def create_dirs(self): - mkdir(self.directory) - mkdir(self.testing_directory) - @property def directory(self): - return join(SOLUTIONS_ROOT, str(self.id)) + return "solutions/" + str(self.id) @property def testing_directory(self): - return join(self.directory, 'test_dir') + return self.directory @property def volume_directory(self): - return join(SOLUTIONS_ROOT_EXTERNAL, str(self.id), 'test_dir') + return "/sprint-data/worker/" + str(self.id) def exec_command(self, command, working_directory='app', timeout=None): return call(f'docker exec -i solution_{self.id} sh -c "cd {working_directory} && {command}"', shell=True, timeout=timeout) diff --git a/Main/models/solution_file.py b/Main/models/solution_file.py new file mode 100644 index 0000000..702b5ff --- /dev/null +++ b/Main/models/solution_file.py @@ -0,0 +1,9 @@ +from django.db import models + +from Main.models.mixins import FileStorageMixin + + +class SolutionFile(FileStorageMixin, models.Model): + path = models.TextField() + fs_id = models.IntegerField() + solution = models.ForeignKey('Solution', on_delete=models.CASCADE) diff --git a/Main/views/TaskSettingsView.py b/Main/views/TaskSettingsView.py index e0998c6..c32a167 100644 --- a/Main/views/TaskSettingsView.py +++ b/Main/views/TaskSettingsView.py @@ -45,11 +45,9 @@ class TaskSettingsView(BaseView): filename=filename, is_test=is_test ) - with open(ef.path, 'wb') as fs: - for chunk in self.request.FILES['file'].chunks(): - fs.write(chunk) + ef.write(self.request.FILES['file'].read()) try: - open(ef.path, 'r').read() + var = ef.text ef.readable = True except UnicodeDecodeError: ef.readable = False @@ -73,8 +71,7 @@ class TaskSettingsView(BaseView): ef, created = ExtraFile.objects.get_or_create(filename=name, task=self.entities.task) if not created: return f'/admin/task?task_id={self.entities.task.id}&error_message=Файл с таким именем уже существует' - with open(ef.path, 'w') as fs: - fs.write('') + ef.write(b"") ef.is_test = is_test ef.readable = True ef.save() @@ -88,8 +85,8 @@ class TaskSettingsView(BaseView): def post_save_test(self): ef = ExtraFile.objects.get(id=self.request.POST['test_id']) - with open(ef.path, 'w') as fs: - fs.write(self.request.POST['text']) + ef.remove_from_fs() + ef.write(self.request.POST['text'].encode('utf-8')) ef.is_sample = 'is_sample' in self.request.POST.keys() ef.save() return f'/admin/task?task_id={self.entities.task.id}' diff --git a/Main/views/TaskView.py b/Main/views/TaskView.py index 5859fea..aeae5e3 100644 --- a/Main/views/TaskView.py +++ b/Main/views/TaskView.py @@ -1,11 +1,11 @@ +import io from zipfile import ZipFile -from os.path import join -from Main.models import Solution, Progress +from Main.models import Solution, Progress, SolutionFile from SprintLib.BaseView import BaseView from SprintLib.language import languages from SprintLib.queue import send_testing -from SprintLib.testers import * +from SprintLib.utils import write_bytes class TaskView(BaseView): @@ -13,40 +13,54 @@ class TaskView(BaseView): view_file = "task.html" def get(self): - self.context['languages'] = sorted(languages, key=lambda x: x.name) - progress, _ = Progress.objects.get_or_create(user=self.request.user, task=self.entities.task) - self.context['progress'] = progress + self.context["languages"] = sorted(languages, key=lambda x: x.name) + progress, _ = Progress.objects.get_or_create( + user=self.request.user, task=self.entities.task + ) + self.context["progress"] = progress def pre_handle(self): - if self.request.method == 'GET': + if self.request.method == "GET": return self.solution = Solution.objects.create( task=self.entities.task, user=self.request.user, - language_id=int(self.request.POST["language"]) + language_id=int(self.request.POST["language"]), ) - self.solution.create_dirs() def post_0(self): # отправка решения через текст - filename = 'solution.' + self.solution.language.file_type - file_path = join(self.solution.directory, filename) - with open(file_path, 'w') as fs: - fs.write(self.request.POST['code']) + fs_id = write_bytes(self.request.POST["code"].encode("utf-8")) + SolutionFile.objects.create( + path="solution." + self.solution.language.file_type, + solution=self.solution, + fs_id=fs_id, + ) send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) def post_1(self): # отправка решения через файл - if 'file' not in self.request.FILES: + if "file" not in self.request.FILES: return "task?task_id=" + str(self.entities.task.id) - filename = self.request.FILES['file'].name - file_path = join(self.solution.directory, filename) - with open(file_path, 'wb') as fs: - for chunk in self.request.FILES['file'].chunks(): - fs.write(chunk) - if filename.endswith('.zip'): - with ZipFile(file_path) as obj: - obj.extractall(self.solution.directory) + filename = self.request.FILES["file"].name + if filename.endswith(".zip"): + archive = ZipFile(io.BytesIO(self.request.FILES['file'].read())) + for file in archive.infolist(): + if file.is_dir(): + continue + fs_id = write_bytes(archive.read(file.filename)) + SolutionFile.objects.create( + path=file.filename, + solution=self.solution, + fs_id=fs_id, + ) + else: + fs_id = write_bytes(self.request.FILES['file'].read()) + SolutionFile.objects.create( + path=filename, + solution=self.solution, + fs_id=fs_id, + ) send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) diff --git a/Sprint/settings.py b/Sprint/settings.py index 217c26e..0ddb1e0 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -146,6 +146,9 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") RABBIT_HOST = HOST RABBIT_PORT = 5672 +FS_HOST = "http://" + HOST +FS_PORT = 5555 + STATICFILES_DIRS = [ os.path.join(BASE_DIR, "Main/static"), ] diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 69b1cf5..74691da 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -1,13 +1,12 @@ from os import listdir, mkdir from os.path import join, exists -from shutil import copyfile, rmtree from subprocess import call, TimeoutExpired from Main.management.commands.bot import bot -from Main.models import ExtraFile +from Main.models import ExtraFile, SolutionFile from Main.models.progress import Progress from Sprint.settings import CONSTS -from SprintLib.utils import copy_content +from SprintLib.utils import get_bytes class TestException(Exception): @@ -56,19 +55,26 @@ class BaseTester: self.solution = solution def execute(self): - if not exists(self.solution.testing_directory): - mkdir(self.solution.testing_directory) - copy_content( - self.solution.directory, self.solution.testing_directory, ("test_dir",) - ) + if not exists("solutions"): + mkdir("solutions") + mkdir("solutions/" + str(self.solution.id)) + for file in SolutionFile.objects.filter(solution=self.solution): + dirs = file.path.split('/') + for i in range(len(dirs) - 1): + name = join(str("solutions/" + self.solution.id), '/'.join(dirs[:i + 1])) + if not exists(name): + mkdir(name) + with open(join("solutions/" + str(self.solution.id), file.path), 'wb') as fs: + fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() - docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" + docker_command = f"docker run --name solution_{self.solution.id} --volume=/sprint-data/solutions/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}" print(docker_command) call(docker_command, shell=True) print("Container created") for file in ExtraFile.objects.filter(task=self.solution.task): - copyfile(file.path, join(self.solution.testing_directory, file.filename)) + with open(join("solutions/" + str(self.solution.id), file.filename), 'wb') as fs: + fs.write(get_bytes(file.fs_id)) print("Files copied") try: self.before_test() @@ -98,7 +104,6 @@ class BaseTester: print(str(e)) self.solution.save() call(f"docker rm --force solution_{self.solution.id}", shell=True) - rmtree(self.solution.testing_directory) self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: bot.send_message( diff --git a/SprintLib/testers/Python3Tester.py b/SprintLib/testers/Python3Tester.py index 1ca8ada..750e3c6 100644 --- a/SprintLib/testers/Python3Tester.py +++ b/SprintLib/testers/Python3Tester.py @@ -1,4 +1,4 @@ -from os import listdir +from os import listdir, getcwd from SprintLib.testers.BaseTester import BaseTester, TestException @@ -7,11 +7,13 @@ class Python3Tester(BaseTester): file = None def before_test(self): + print(getcwd()) for file in listdir(self.solution.testing_directory): if file.endswith(".py"): self.file = file break if self.file is None: + print('no file') raise TestException("TE") @property diff --git a/SprintLib/utils.py b/SprintLib/utils.py index df682d3..751538f 100644 --- a/SprintLib/utils.py +++ b/SprintLib/utils.py @@ -1,12 +1,21 @@ -from os import listdir -from os.path import isfile, join -from shutil import copyfile, copytree +from requests import get, post + +from Sprint import settings -def copy_content(from_dir, to_dir, exc=()): - for file in listdir(from_dir): - if file in exc: - continue - full_path = join(from_dir, file) - func = copyfile if isfile(full_path) else copytree - func(full_path, join(to_dir, file)) +def write_bytes(data): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file" + print(url) + return post(url, data=data).json()['id'] + + +def get_bytes(num): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num) + print(url) + return get(url).content + + +def delete_file(num): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num) + print(url) + post(url) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9a99246..1e37b0f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,11 +27,20 @@ services: ports: - "${PORT}:${PORT}" volumes: - - /sprint-data/data:/usr/src/app/data - /sprint-data/media:/usr/src/app/media depends_on: - postgres - rabbitmq + - storage + + storage: + restart: always + image: mathwave/sprint-repo:sprint + command: python manage.py storage + ports: + - "5555:5555" + volumes: + - /sprint-data/data:/usr/src/app/data bot: image: mathwave/sprint-repo:sprint @@ -63,6 +72,7 @@ services: - web - rabbitmq - postgres + - storage volumes: - - /sprint-data/data:/usr/src/app/data + - /sprint-data/solutions:/usr/src/app/solutions - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index dbd0f50..d6566bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +aiofiles==0.7.0 +aiohttp==3.8.0 amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0 diff --git a/scripts/runserver.sh b/scripts/runserver.sh index b56d668..2592052 100755 --- a/scripts/runserver.sh +++ b/scripts/runserver.sh @@ -1,3 +1,2 @@ python manage.py migrate -python manage.py update_languages python manage.py runserver 0.0.0.0:$PORT --noreload \ No newline at end of file