From edb58e23a32480bc1c38cffab95ee5dd18817476 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 11:47:06 +0300 Subject: [PATCH 01/24] filestorage --- .gitlab-ci.yml | 1 + FileStorage/__init__.py | 0 FileStorage/root.py | 21 +++++++++++ FileStorage/routes.py | 9 +++++ FileStorage/sync.py | 26 +++++++++++++ FileStorage/views/__init__.py | 3 ++ FileStorage/views/delete_file.py | 8 ++++ FileStorage/views/get_file.py | 10 +++++ FileStorage/views/upload_file.py | 11 ++++++ Main/management/commands/storage.py | 9 +++++ Main/models/__init__.py | 1 + Main/models/extrafile.py | 21 +++-------- Main/models/mixins.py | 14 +++++++ Main/models/solution.py | 49 +++++++++++------------- Main/models/solution_file.py | 9 +++++ Main/views/TaskSettingsView.py | 13 +++---- Main/views/TaskView.py | 58 ++++++++++++++++++----------- Sprint/settings.py | 3 ++ SprintLib/testers/BaseTester.py | 23 +++++++----- SprintLib/utils.py | 29 ++++++++++----- docker-compose.yaml | 10 +++++ requirements.txt | 2 + 22 files changed, 237 insertions(+), 93 deletions(-) create mode 100644 FileStorage/__init__.py create mode 100644 FileStorage/root.py create mode 100644 FileStorage/routes.py create mode 100644 FileStorage/sync.py create mode 100644 FileStorage/views/__init__.py create mode 100644 FileStorage/views/delete_file.py create mode 100644 FileStorage/views/get_file.py create mode 100644 FileStorage/views/upload_file.py create mode 100644 Main/management/commands/storage.py create mode 100644 Main/models/mixins.py create mode 100644 Main/models/solution_file.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f990870..391b7bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,7 @@ deploy-dev: SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" RABBIT_HOST: "rabbitmq" + FS_HOST: "storage" deploy-prod: extends: diff --git a/FileStorage/__init__.py b/FileStorage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/FileStorage/root.py b/FileStorage/root.py new file mode 100644 index 0000000..3541645 --- /dev/null +++ b/FileStorage/root.py @@ -0,0 +1,21 @@ +import os +from os import mkdir +from os.path import exists + +from aiohttp import web +from FileStorage.routes import setup_routes + + +def runserver(): + app = web.Application() + setup_routes(app) + if not exists("data"): + mkdir("data") + if not exists("data/meta.txt"): + with open("data/meta.txt", "w") as fs: + fs.write("0") + web.run_app(app, host=os.getenv("FS_HOST", "0.0.0.0"), port=5555) + + +if __name__ == "__main__": + runserver() diff --git a/FileStorage/routes.py b/FileStorage/routes.py new file mode 100644 index 0000000..3ce9fcf --- /dev/null +++ b/FileStorage/routes.py @@ -0,0 +1,9 @@ +from aiohttp import web + +from FileStorage.views import get_file, upload_file, delete_file + + +def setup_routes(app: web.Application): + app.router.add_get("/get_file", get_file) + app.router.add_post("/upload_file", upload_file) + app.router.add_post("/delete_file", delete_file) diff --git a/FileStorage/sync.py b/FileStorage/sync.py new file mode 100644 index 0000000..fd82ce3 --- /dev/null +++ b/FileStorage/sync.py @@ -0,0 +1,26 @@ +import threading + +import aiofiles + + +def synchronized_method(method): + outer_lock = threading.Lock() + lock_name = "__" + method.__name__ + "_lock" + "__" + + def sync_method(self, *args, **kws): + with outer_lock: + if not hasattr(self, lock_name): + setattr(self, lock_name, threading.Lock()) + lock = getattr(self, lock_name) + with lock: + return method(self, *args, **kws) + return sync_method + + +@synchronized_method +async def write_meta(request): + async with aiofiles.open("data/meta.txt", "r") as fs: + num = int(await fs.read()) + 1 + async with aiofiles.open("data/meta.txt", "w") as fs: + await fs.write(str(num)) + return num diff --git a/FileStorage/views/__init__.py b/FileStorage/views/__init__.py new file mode 100644 index 0000000..2ecea32 --- /dev/null +++ b/FileStorage/views/__init__.py @@ -0,0 +1,3 @@ +from .get_file import get_file +from .upload_file import upload_file +from .delete_file import delete_file diff --git a/FileStorage/views/delete_file.py b/FileStorage/views/delete_file.py new file mode 100644 index 0000000..c581cf6 --- /dev/null +++ b/FileStorage/views/delete_file.py @@ -0,0 +1,8 @@ +from os import remove + +from aiohttp import web + + +async def delete_file(request): + remove("data/" + request.rel_url.query['id']) + return web.json_response({"success": True}) diff --git a/FileStorage/views/get_file.py b/FileStorage/views/get_file.py new file mode 100644 index 0000000..e89cc2a --- /dev/null +++ b/FileStorage/views/get_file.py @@ -0,0 +1,10 @@ +import aiofiles +from aiohttp import web + + +async def get_file(request): + response = web.StreamResponse() + await response.prepare(request) + async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs: + await response.write_eof(await fs.read()) + return response diff --git a/FileStorage/views/upload_file.py b/FileStorage/views/upload_file.py new file mode 100644 index 0000000..c5f2ff6 --- /dev/null +++ b/FileStorage/views/upload_file.py @@ -0,0 +1,11 @@ +from aiohttp import web + +from FileStorage.sync import write_meta +import aiofiles + + +async def upload_file(request): + file_id = await write_meta(request) + async with aiofiles.open("data/" + str(file_id), "wb") as fs: + await fs.write(await request.content.read()) + return web.json_response({"id": file_id}) diff --git a/Main/management/commands/storage.py b/Main/management/commands/storage.py new file mode 100644 index 0000000..1263566 --- /dev/null +++ b/Main/management/commands/storage.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from FileStorage.root import runserver + + +class Command(BaseCommand): + help = 'starts FileStorage' + + def handle(self, *args, **options): + runserver() diff --git a/Main/models/__init__.py b/Main/models/__init__.py index 393b604..36d607c 100644 --- a/Main/models/__init__.py +++ b/Main/models/__init__.py @@ -8,3 +8,4 @@ from Main.models.settask import SetTask from Main.models.solution import Solution from Main.models.extrafile import ExtraFile from Main.models.progress import Progress +from Main.models.solution_file import SolutionFile diff --git a/Main/models/extrafile.py b/Main/models/extrafile.py index bb7c511..c2895a3 100644 --- a/Main/models/extrafile.py +++ b/Main/models/extrafile.py @@ -1,23 +1,17 @@ -from os import remove -from os.path import join, exists - from django.core.exceptions import ObjectDoesNotExist from django.db import models -from Sprint.settings import DATA_ROOT +from .mixins import FileStorageMixin -class ExtraFile(models.Model): +class ExtraFile(FileStorageMixin, models.Model): task = models.ForeignKey("Task", on_delete=models.CASCADE) filename = models.TextField() is_test = models.BooleanField(null=True) is_sample = models.BooleanField(null=True) readable = models.BooleanField(null=True) test_number = models.IntegerField(null=True) - - @property - def path(self): - return join(DATA_ROOT, "extra_files", str(self.id)) + fs_id = models.IntegerField(null=True) @property def can_be_sample(self): @@ -29,13 +23,8 @@ class ExtraFile(models.Model): ) ) - @property - def text(self): - return open(self.path, "r").read() - def delete(self, using=None, keep_parents=False): - if exists(self.path): - remove(self.path) + self.remove_from_fs() if self.is_test and self.filename.endswith('.a'): try: ef = ExtraFile.objects.get(task=self.task, filename=self.filename.rstrip('.a'), is_test=True) @@ -47,4 +36,4 @@ class ExtraFile(models.Model): @property def answer(self): - return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a') \ No newline at end of file + return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a') diff --git a/Main/models/mixins.py b/Main/models/mixins.py new file mode 100644 index 0000000..1b3aba7 --- /dev/null +++ b/Main/models/mixins.py @@ -0,0 +1,14 @@ +from SprintLib.utils import get_bytes, write_bytes, delete_file + + +class FileStorageMixin: + @property + def text(self): + return get_bytes(self.fs_id).decode("utf-8") + + def write(self, bytes): + self.fs_id = write_bytes(bytes) + self.save() + + def remove_from_fs(self): + delete_file(self.fs_id) diff --git a/Main/models/solution.py b/Main/models/solution.py index 52256ad..9e365dd 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -1,13 +1,12 @@ -from os import mkdir, walk from os.path import join, exists from shutil import rmtree from subprocess import call from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.utils import timezone +from Main.models.solution_file import SolutionFile from Main.models.task import Task from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL from SprintLib.language import languages @@ -32,36 +31,30 @@ class Solution(models.Model): @property def files(self): data = [] - for path, _, files in walk(self.directory): - if path.startswith(self.testing_directory): + for file in SolutionFile.objects.filter(solution=self): + try: + text = file.text + except: continue - for file in files: - try: - entity = { - 'filename': file, - 'text': open(join(path, file), 'r').read() - } - end = file.split('.')[-1] - language = None - for l in languages: - if l.file_type == end: - language = l - break - if language is None: - highlight = 'nohighlight' - else: - highlight = 'language-' + language.highlight - entity['highlight'] = highlight - data.append(entity) - except: - continue + entity = { + 'filename': file.path, + 'text': text + } + end = file.path.split('.')[-1] + language = None + for l in languages: + if l.file_type == end: + language = l + break + if language is None: + highlight = 'nohighlight' + else: + highlight = 'language-' + language.highlight + entity['highlight'] = highlight + data.append(entity) data.sort(key=lambda x: x['filename']) return data - def create_dirs(self): - mkdir(self.directory) - mkdir(self.testing_directory) - @property def directory(self): return join(SOLUTIONS_ROOT, str(self.id)) diff --git a/Main/models/solution_file.py b/Main/models/solution_file.py new file mode 100644 index 0000000..702b5ff --- /dev/null +++ b/Main/models/solution_file.py @@ -0,0 +1,9 @@ +from django.db import models + +from Main.models.mixins import FileStorageMixin + + +class SolutionFile(FileStorageMixin, models.Model): + path = models.TextField() + fs_id = models.IntegerField() + solution = models.ForeignKey('Solution', on_delete=models.CASCADE) diff --git a/Main/views/TaskSettingsView.py b/Main/views/TaskSettingsView.py index e0998c6..c32a167 100644 --- a/Main/views/TaskSettingsView.py +++ b/Main/views/TaskSettingsView.py @@ -45,11 +45,9 @@ class TaskSettingsView(BaseView): filename=filename, is_test=is_test ) - with open(ef.path, 'wb') as fs: - for chunk in self.request.FILES['file'].chunks(): - fs.write(chunk) + ef.write(self.request.FILES['file'].read()) try: - open(ef.path, 'r').read() + var = ef.text ef.readable = True except UnicodeDecodeError: ef.readable = False @@ -73,8 +71,7 @@ class TaskSettingsView(BaseView): ef, created = ExtraFile.objects.get_or_create(filename=name, task=self.entities.task) if not created: return f'/admin/task?task_id={self.entities.task.id}&error_message=Файл с таким именем уже существует' - with open(ef.path, 'w') as fs: - fs.write('') + ef.write(b"") ef.is_test = is_test ef.readable = True ef.save() @@ -88,8 +85,8 @@ class TaskSettingsView(BaseView): def post_save_test(self): ef = ExtraFile.objects.get(id=self.request.POST['test_id']) - with open(ef.path, 'w') as fs: - fs.write(self.request.POST['text']) + ef.remove_from_fs() + ef.write(self.request.POST['text'].encode('utf-8')) ef.is_sample = 'is_sample' in self.request.POST.keys() ef.save() return f'/admin/task?task_id={self.entities.task.id}' diff --git a/Main/views/TaskView.py b/Main/views/TaskView.py index 5859fea..aeae5e3 100644 --- a/Main/views/TaskView.py +++ b/Main/views/TaskView.py @@ -1,11 +1,11 @@ +import io from zipfile import ZipFile -from os.path import join -from Main.models import Solution, Progress +from Main.models import Solution, Progress, SolutionFile from SprintLib.BaseView import BaseView from SprintLib.language import languages from SprintLib.queue import send_testing -from SprintLib.testers import * +from SprintLib.utils import write_bytes class TaskView(BaseView): @@ -13,40 +13,54 @@ class TaskView(BaseView): view_file = "task.html" def get(self): - self.context['languages'] = sorted(languages, key=lambda x: x.name) - progress, _ = Progress.objects.get_or_create(user=self.request.user, task=self.entities.task) - self.context['progress'] = progress + self.context["languages"] = sorted(languages, key=lambda x: x.name) + progress, _ = Progress.objects.get_or_create( + user=self.request.user, task=self.entities.task + ) + self.context["progress"] = progress def pre_handle(self): - if self.request.method == 'GET': + if self.request.method == "GET": return self.solution = Solution.objects.create( task=self.entities.task, user=self.request.user, - language_id=int(self.request.POST["language"]) + language_id=int(self.request.POST["language"]), ) - self.solution.create_dirs() def post_0(self): # отправка решения через текст - filename = 'solution.' + self.solution.language.file_type - file_path = join(self.solution.directory, filename) - with open(file_path, 'w') as fs: - fs.write(self.request.POST['code']) + fs_id = write_bytes(self.request.POST["code"].encode("utf-8")) + SolutionFile.objects.create( + path="solution." + self.solution.language.file_type, + solution=self.solution, + fs_id=fs_id, + ) send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) def post_1(self): # отправка решения через файл - if 'file' not in self.request.FILES: + if "file" not in self.request.FILES: return "task?task_id=" + str(self.entities.task.id) - filename = self.request.FILES['file'].name - file_path = join(self.solution.directory, filename) - with open(file_path, 'wb') as fs: - for chunk in self.request.FILES['file'].chunks(): - fs.write(chunk) - if filename.endswith('.zip'): - with ZipFile(file_path) as obj: - obj.extractall(self.solution.directory) + filename = self.request.FILES["file"].name + if filename.endswith(".zip"): + archive = ZipFile(io.BytesIO(self.request.FILES['file'].read())) + for file in archive.infolist(): + if file.is_dir(): + continue + fs_id = write_bytes(archive.read(file.filename)) + SolutionFile.objects.create( + path=file.filename, + solution=self.solution, + fs_id=fs_id, + ) + else: + fs_id = write_bytes(self.request.FILES['file'].read()) + SolutionFile.objects.create( + path=filename, + solution=self.solution, + fs_id=fs_id, + ) send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) diff --git a/Sprint/settings.py b/Sprint/settings.py index 4ea8baa..18847a2 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -143,6 +143,9 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") RABBIT_HOST = os.getenv("RABBIT_HOST", "0.0.0.0") RABBIT_PORT = 5672 +FS_HOST = os.getenv("FS_HOST", "http://0.0.0.0") +FS_PORT = 5555 + STATICFILES_DIRS = [ os.path.join(BASE_DIR, "Main/static"), ] diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 69b1cf5..7e763c7 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -4,10 +4,10 @@ from shutil import copyfile, rmtree from subprocess import call, TimeoutExpired from Main.management.commands.bot import bot -from Main.models import ExtraFile +from Main.models import ExtraFile, SolutionFile from Main.models.progress import Progress from Sprint.settings import CONSTS -from SprintLib.utils import copy_content +from SprintLib.utils import get_bytes class TestException(Exception): @@ -56,11 +56,15 @@ class BaseTester: self.solution = solution def execute(self): - if not exists(self.solution.testing_directory): - mkdir(self.solution.testing_directory) - copy_content( - self.solution.directory, self.solution.testing_directory, ("test_dir",) - ) + mkdir("solution") + for file in SolutionFile.objects.filter(solution=self.solution): + dirs = file.path.split('/') + for i in range(len(dirs) - 1): + name = join("solution", '/'.join(dirs[:i + 1])) + if not exists(name): + mkdir(name) + with open(file.path, 'wb') as fs: + fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" @@ -68,7 +72,8 @@ class BaseTester: call(docker_command, shell=True) print("Container created") for file in ExtraFile.objects.filter(task=self.solution.task): - copyfile(file.path, join(self.solution.testing_directory, file.filename)) + with open(join("solution", file.filename), 'wb') as fs: + fs.write(get_bytes(file.fs_id)) print("Files copied") try: self.before_test() @@ -98,7 +103,7 @@ class BaseTester: print(str(e)) self.solution.save() call(f"docker rm --force solution_{self.solution.id}", shell=True) - rmtree(self.solution.testing_directory) + rmtree("solution") self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: bot.send_message( diff --git a/SprintLib/utils.py b/SprintLib/utils.py index df682d3..751538f 100644 --- a/SprintLib/utils.py +++ b/SprintLib/utils.py @@ -1,12 +1,21 @@ -from os import listdir -from os.path import isfile, join -from shutil import copyfile, copytree +from requests import get, post + +from Sprint import settings -def copy_content(from_dir, to_dir, exc=()): - for file in listdir(from_dir): - if file in exc: - continue - full_path = join(from_dir, file) - func = copyfile if isfile(full_path) else copytree - func(full_path, join(to_dir, file)) +def write_bytes(data): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file" + print(url) + return post(url, data=data).json()['id'] + + +def get_bytes(num): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num) + print(url) + return get(url).content + + +def delete_file(num): + url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num) + print(url) + post(url) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0eaa7eb..1a65cce 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,6 +24,7 @@ services: PORT: $PORT DB_HOST: $DB_HOST RABBIT_HOST: $RABBIT_HOST + FS_HOST: $FS_HOST command: scripts/runserver.sh ports: - "${PORT}:${PORT}" @@ -33,6 +34,15 @@ services: depends_on: - postgres - rabbitmq + - storage + + storage: + restart: always + image: mathwave/sprint-repo:sprint + ports: + - "5555:5555" + volumes: + - /sprint-data/data:/usr/src/app/FileStorage/data bot: image: mathwave/sprint-repo:sprint diff --git a/requirements.txt b/requirements.txt index dbd0f50..d6566bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +aiofiles==0.7.0 +aiohttp==3.8.0 amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0 From e70c94bfe2b785038f526bdd443b0784f35b094e Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 11:55:20 +0300 Subject: [PATCH 02/24] filestorage --- Main/migrations/0075_auto_20211110_2317.py | 31 ++++++++++++++++++++++ docker-compose.yaml | 2 -- scripts/runserver.sh | 1 - 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 Main/migrations/0075_auto_20211110_2317.py diff --git a/Main/migrations/0075_auto_20211110_2317.py b/Main/migrations/0075_auto_20211110_2317.py new file mode 100644 index 0000000..0c4d95d --- /dev/null +++ b/Main/migrations/0075_auto_20211110_2317.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.4 on 2021-11-10 20:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('Main', '0074_auto_20211106_1215'), + ] + + operations = [ + migrations.CreateModel( + name='SolutionFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.TextField()), + ('fs_id', models.IntegerField()), + ('solution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='Main.solution')), + ], + ), + migrations.DeleteModel( + name='Language', + ), + migrations.AddField( + model_name='extrafile', + name='fs_id', + field=models.IntegerField(null=True), + ), + ] diff --git a/docker-compose.yaml b/docker-compose.yaml index 1a65cce..1989236 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -29,7 +29,6 @@ services: ports: - "${PORT}:${PORT}" volumes: - - /sprint-data/data:/usr/src/app/data - /sprint-data/media:/usr/src/app/media depends_on: - postgres @@ -74,5 +73,4 @@ services: - rabbitmq - postgres volumes: - - /sprint-data/data:/usr/src/app/data - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/scripts/runserver.sh b/scripts/runserver.sh index b56d668..2592052 100755 --- a/scripts/runserver.sh +++ b/scripts/runserver.sh @@ -1,3 +1,2 @@ python manage.py migrate -python manage.py update_languages python manage.py runserver 0.0.0.0:$PORT --noreload \ No newline at end of file From 5bd7b68006f3d08d4b158f05403d831f84132bea Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 12:02:08 +0300 Subject: [PATCH 03/24] http fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 391b7bf..9b71b67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ deploy-dev: SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" RABBIT_HOST: "rabbitmq" - FS_HOST: "storage" + FS_HOST: "http://storage" deploy-prod: extends: From 84a29028c4dda19063efdc907e6a5eb813bfc87f Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 12:16:00 +0300 Subject: [PATCH 04/24] fix --- .gitlab-ci.yml | 2 +- docker-compose.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b71b67..391b7bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ deploy-dev: SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" RABBIT_HOST: "rabbitmq" - FS_HOST: "http://storage" + FS_HOST: "storage" deploy-prod: extends: diff --git a/docker-compose.yaml b/docker-compose.yaml index 1989236..5c966bc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -38,10 +38,11 @@ services: storage: restart: always image: mathwave/sprint-repo:sprint + command: python manage.py storage ports: - "5555:5555" volumes: - - /sprint-data/data:/usr/src/app/FileStorage/data + - /sprint-data/data:/usr/src/app/data bot: image: mathwave/sprint-repo:sprint From ccb9e1f7d03f110509225c2a857eb37874311a4e Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 12:22:03 +0300 Subject: [PATCH 05/24] fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 391b7bf..9b71b67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ deploy-dev: SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" RABBIT_HOST: "rabbitmq" - FS_HOST: "storage" + FS_HOST: "http://storage" deploy-prod: extends: From 33a896063424ce892dba229108e0a4adb3396ada Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 12:32:16 +0300 Subject: [PATCH 06/24] depends --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5c966bc..9fdf219 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -73,5 +73,6 @@ services: - web - rabbitmq - postgres + - storage volumes: - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file From 6fa641d9cadb88407b908f30425c704bcec25ca5 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 12:42:27 +0300 Subject: [PATCH 07/24] depends --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9fdf219..12cc781 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -69,6 +69,7 @@ services: SOLUTIONS_ROOT_EXTERNAL: "${SOLUTIONS_ROOT_EXTERNAL}" DB_HOST: "${DB_HOST}" RABBIT_HOST: "${RABBIT_HOST}" + FS_HOST: "${FS_HOST}" depends_on: - web - rabbitmq From 1600479cb73219cf0757dbb41c287143e8ec1ccd Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 13:12:15 +0300 Subject: [PATCH 08/24] volume --- Main/models/solution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Main/models/solution.py b/Main/models/solution.py index 9e365dd..eb8b0c4 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -65,7 +65,7 @@ class Solution(models.Model): @property def volume_directory(self): - return join(SOLUTIONS_ROOT_EXTERNAL, str(self.id), 'test_dir') + return "solution" def exec_command(self, command, working_directory='app', timeout=None): return call(f'docker exec -i solution_{self.id} sh -c "cd {working_directory} && {command}"', shell=True, timeout=timeout) From e7f63094873a5db983f37fdd5540268fff7e0e88 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 13:29:49 +0300 Subject: [PATCH 09/24] volume --- Main/models/solution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Main/models/solution.py b/Main/models/solution.py index eb8b0c4..5032cd9 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -57,11 +57,11 @@ class Solution(models.Model): @property def directory(self): - return join(SOLUTIONS_ROOT, str(self.id)) + return "solution" @property def testing_directory(self): - return join(self.directory, 'test_dir') + return self.directory @property def volume_directory(self): From 667d33a20b867f39c4fa476b7b2b2d93322b7d7e Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 13:47:41 +0300 Subject: [PATCH 10/24] volume --- SprintLib/testers/BaseTester.py | 1 - 1 file changed, 1 deletion(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 7e763c7..a1f1549 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -103,7 +103,6 @@ class BaseTester: print(str(e)) self.solution.save() call(f"docker rm --force solution_{self.solution.id}", shell=True) - rmtree("solution") self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: bot.send_message( From 1293d6ee2db6220cdc9101f89787e9e69f081bc3 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 14:29:04 +0300 Subject: [PATCH 11/24] exc --- SprintLib/testers/BaseTester.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index a1f1549..9be7878 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -94,8 +94,6 @@ class BaseTester: progress.finished = True progress.save() progress.increment_rating() - except TestException as e: - self.solution.result = str(e) except TimeoutExpired: self.solution.result = "TL" except Exception as e: From 3cbf1d67d16ff41478aaabf1b8202b6ab2c0f7c9 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 14:44:42 +0300 Subject: [PATCH 12/24] path --- SprintLib/testers/BaseTester.py | 2 +- SprintLib/testers/Python3Tester.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 9be7878..ccea1a3 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -63,7 +63,7 @@ class BaseTester: name = join("solution", '/'.join(dirs[:i + 1])) if not exists(name): mkdir(name) - with open(file.path, 'wb') as fs: + with open(join("solution", file.path), 'wb') as fs: fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() diff --git a/SprintLib/testers/Python3Tester.py b/SprintLib/testers/Python3Tester.py index 1ca8ada..d90948e 100644 --- a/SprintLib/testers/Python3Tester.py +++ b/SprintLib/testers/Python3Tester.py @@ -12,6 +12,7 @@ class Python3Tester(BaseTester): self.file = file break if self.file is None: + print('no file') raise TestException("TE") @property From 3ee46addfcdf15bfa94f39c96b4f59f16b2b78f9 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Thu, 11 Nov 2021 14:55:16 +0300 Subject: [PATCH 13/24] path --- SprintLib/testers/BaseTester.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index ccea1a3..902fc89 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -94,6 +94,8 @@ class BaseTester: progress.finished = True progress.save() progress.increment_rating() + except TestException as e: + self.solution.result = str(e) except TimeoutExpired: self.solution.result = "TL" except Exception as e: From d5a7b3a21585411bc1ff5d8e788beec1e267b4c8 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Fri, 12 Nov 2021 00:11:00 +0300 Subject: [PATCH 14/24] path --- SprintLib/testers/Python3Tester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SprintLib/testers/Python3Tester.py b/SprintLib/testers/Python3Tester.py index d90948e..750e3c6 100644 --- a/SprintLib/testers/Python3Tester.py +++ b/SprintLib/testers/Python3Tester.py @@ -1,4 +1,4 @@ -from os import listdir +from os import listdir, getcwd from SprintLib.testers.BaseTester import BaseTester, TestException @@ -7,6 +7,7 @@ class Python3Tester(BaseTester): file = None def before_test(self): + print(getcwd()) for file in listdir(self.solution.testing_directory): if file.endswith(".py"): self.file = file From c249de1dd481d28455dc38376e551f4ca5d525f6 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Fri, 12 Nov 2021 00:27:29 +0300 Subject: [PATCH 15/24] path --- SprintLib/testers/BaseTester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 902fc89..66a200c 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -67,7 +67,7 @@ class BaseTester: fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() - docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" + docker_command = f"docker run --name solution_{self.solution.id} --volume=/usr/src/app/solution:/{self.working_directory} -t -d {self.solution.language.image}" print(docker_command) call(docker_command, shell=True) print("Container created") From a31d940429b45e0136b1e30d313e9aec2308dd77 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Fri, 12 Nov 2021 00:33:58 +0300 Subject: [PATCH 16/24] path --- SprintLib/testers/BaseTester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 66a200c..646e52f 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -102,7 +102,7 @@ class BaseTester: self.solution.result = "TE" print(str(e)) self.solution.save() - call(f"docker rm --force solution_{self.solution.id}", shell=True) + # call(f"docker rm --force solution_{self.solution.id}", shell=True) self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: bot.send_message( From cc95eb34ecceca2adcafaf90e606f8282cb8c0ff Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Fri, 12 Nov 2021 00:53:16 +0300 Subject: [PATCH 17/24] path --- Main/models/solution.py | 2 +- SprintLib/testers/BaseTester.py | 10 +++++----- docker-compose.yaml | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Main/models/solution.py b/Main/models/solution.py index 5032cd9..3629208 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -65,7 +65,7 @@ class Solution(models.Model): @property def volume_directory(self): - return "solution" + return "/sprint-data/worker/" + str(self.id) def exec_command(self, command, working_directory='app', timeout=None): return call(f'docker exec -i solution_{self.id} sh -c "cd {working_directory} && {command}"', shell=True, timeout=timeout) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 646e52f..405159a 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -60,19 +60,19 @@ class BaseTester: for file in SolutionFile.objects.filter(solution=self.solution): dirs = file.path.split('/') for i in range(len(dirs) - 1): - name = join("solution", '/'.join(dirs[:i + 1])) + name = join(str(self.solution.id), '/'.join(dirs[:i + 1])) if not exists(name): mkdir(name) - with open(join("solution", file.path), 'wb') as fs: + with open(join(str(self.solution.id), file.path), 'wb') as fs: fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() - docker_command = f"docker run --name solution_{self.solution.id} --volume=/usr/src/app/solution:/{self.working_directory} -t -d {self.solution.language.image}" + docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" print(docker_command) call(docker_command, shell=True) print("Container created") for file in ExtraFile.objects.filter(task=self.solution.task): - with open(join("solution", file.filename), 'wb') as fs: + with open(join(str(self.solution.id), file.filename), 'wb') as fs: fs.write(get_bytes(file.fs_id)) print("Files copied") try: @@ -102,7 +102,7 @@ class BaseTester: self.solution.result = "TE" print(str(e)) self.solution.save() - # call(f"docker rm --force solution_{self.solution.id}", shell=True) + call(f"docker rm --force solution_{self.solution.id}", shell=True) self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: bot.send_message( diff --git a/docker-compose.yaml b/docker-compose.yaml index 12cc781..0267d38 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,4 +76,5 @@ services: - postgres - storage volumes: + - /sprint-data/worker:/usr/src/app - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file From 69b680864d6b1771d24fdfcecee49d10431f3745 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 17:23:21 +0300 Subject: [PATCH 18/24] docker --- docker-compose.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 682ca08..3641e6f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -22,13 +22,7 @@ services: restart: always environment: PORT: $PORT -<<<<<<< HEAD - DB_HOST: $DB_HOST - RABBIT_HOST: $RABBIT_HOST - FS_HOST: $FS_HOST -======= HOST: $HOST ->>>>>>> a512261c4d4b443cf25874181acd50485f1b40a4 command: scripts/runserver.sh ports: - "${PORT}:${PORT}" From f912abc9b56e2bb92159d5852726ef9d2a7eb8b2 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 17:29:56 +0300 Subject: [PATCH 19/24] volumes --- SprintLib/testers/BaseTester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 405159a..7d82893 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -56,7 +56,7 @@ class BaseTester: self.solution = solution def execute(self): - mkdir("solution") + mkdir(str(self.solution.id)) for file in SolutionFile.objects.filter(solution=self.solution): dirs = file.path.split('/') for i in range(len(dirs) - 1): @@ -67,7 +67,7 @@ class BaseTester: fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() - docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" + docker_command = f"docker run --name solution_{self.solution.id} --volume=/sprint-data/worker/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}" print(docker_command) call(docker_command, shell=True) print("Container created") From ead4598b38795bc7521e6bc1f08c727e658af649 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 17:30:42 +0300 Subject: [PATCH 20/24] host --- Sprint/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sprint/settings.py b/Sprint/settings.py index d0657dc..9af2e57 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -146,7 +146,7 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") RABBIT_HOST = HOST RABBIT_PORT = 5672 -FS_HOST = os.getenv("FS_HOST", "http://0.0.0.0") +FS_HOST = HOST FS_PORT = 5555 STATICFILES_DIRS = [ From 7ecc725dbbda6924864364c185aef33be4ddf509 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 17:43:15 +0300 Subject: [PATCH 21/24] http --- Sprint/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sprint/settings.py b/Sprint/settings.py index 9af2e57..0ddb1e0 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -146,7 +146,7 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") RABBIT_HOST = HOST RABBIT_PORT = 5672 -FS_HOST = HOST +FS_HOST = "http://" + HOST FS_PORT = 5555 STATICFILES_DIRS = [ From 8b772f25bbbb587db62679cd3ab2f70d48bf2456 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 18:05:37 +0300 Subject: [PATCH 22/24] http --- SprintLib/testers/BaseTester.py | 6 ++++-- docker-compose.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 7d82893..6930c75 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -56,7 +56,9 @@ class BaseTester: self.solution = solution def execute(self): - mkdir(str(self.solution.id)) + if not exists("solutions"): + mkdir("solutions") + mkdir("solutions/" + str(self.solution.id)) for file in SolutionFile.objects.filter(solution=self.solution): dirs = file.path.split('/') for i in range(len(dirs) - 1): @@ -67,7 +69,7 @@ class BaseTester: fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() - docker_command = f"docker run --name solution_{self.solution.id} --volume=/sprint-data/worker/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}" + docker_command = f"docker run --name solution_{self.solution.id} --volume=/sprint-data/solutions/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}" print(docker_command) call(docker_command, shell=True) print("Container created") diff --git a/docker-compose.yaml b/docker-compose.yaml index 3641e6f..1e37b0f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -74,5 +74,5 @@ services: - postgres - storage volumes: - - /sprint-data/worker:/usr/src/app + - /sprint-data/solutions:/usr/src/app/solutions - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file From cc3d9b409f0c8bfb1a6f9001dd66020595c49215 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 18:15:37 +0300 Subject: [PATCH 23/24] http --- SprintLib/testers/BaseTester.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 6930c75..74691da 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -1,6 +1,5 @@ from os import listdir, mkdir from os.path import join, exists -from shutil import copyfile, rmtree from subprocess import call, TimeoutExpired from Main.management.commands.bot import bot @@ -62,10 +61,10 @@ class BaseTester: for file in SolutionFile.objects.filter(solution=self.solution): dirs = file.path.split('/') for i in range(len(dirs) - 1): - name = join(str(self.solution.id), '/'.join(dirs[:i + 1])) + name = join(str("solutions/" + self.solution.id), '/'.join(dirs[:i + 1])) if not exists(name): mkdir(name) - with open(join(str(self.solution.id), file.path), 'wb') as fs: + with open(join("solutions/" + str(self.solution.id), file.path), 'wb') as fs: fs.write(get_bytes(file.fs_id)) self.solution.result = CONSTS["testing_status"] self.solution.save() @@ -74,7 +73,7 @@ class BaseTester: call(docker_command, shell=True) print("Container created") for file in ExtraFile.objects.filter(task=self.solution.task): - with open(join(str(self.solution.id), file.filename), 'wb') as fs: + with open(join("solutions/" + str(self.solution.id), file.filename), 'wb') as fs: fs.write(get_bytes(file.fs_id)) print("Files copied") try: From bf9dbd3d5edb9bfc8019bfb1269b4036d2a20a10 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sat, 13 Nov 2021 18:25:04 +0300 Subject: [PATCH 24/24] http --- Main/models/solution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Main/models/solution.py b/Main/models/solution.py index 3629208..8c1a491 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -57,7 +57,7 @@ class Solution(models.Model): @property def directory(self): - return "solution" + return "solutions/" + str(self.id) @property def testing_directory(self):