Merge branch 'files' into 'master'

Files

See merge request mathwave/sprint!7

FileStorage/root.py

@@ -0,0 +1,21 @@
+import os
+from os import mkdir
+from os.path import exists
+
+from aiohttp import web
+from FileStorage.routes import setup_routes
+
+
+def runserver():
+    app = web.Application()
+    setup_routes(app)
+    if not exists("data"):
+        mkdir("data")
+    if not exists("data/meta.txt"):
+        with open("data/meta.txt", "w") as fs:
+            fs.write("0")
+    web.run_app(app, host=os.getenv("FS_HOST", "0.0.0.0"), port=5555)
+
+
+if __name__ == "__main__":
+    runserver()

FileStorage/routes.py

@@ -0,0 +1,9 @@
+from aiohttp import web
+
+from FileStorage.views import get_file, upload_file, delete_file
+
+
+def setup_routes(app: web.Application):
+    app.router.add_get("/get_file", get_file)
+    app.router.add_post("/upload_file", upload_file)
+    app.router.add_post("/delete_file", delete_file)

FileStorage/sync.py

@@ -0,0 +1,26 @@
+import threading
+
+import aiofiles
+
+
+def synchronized_method(method):
+    outer_lock = threading.Lock()
+    lock_name = "__" + method.__name__ + "_lock" + "__"
+
+    def sync_method(self, *args, **kws):
+        with outer_lock:
+            if not hasattr(self, lock_name):
+                setattr(self, lock_name, threading.Lock())
+            lock = getattr(self, lock_name)
+            with lock:
+                return method(self, *args, **kws)
+    return sync_method
+
+
+@synchronized_method
+async def write_meta(request):
+    async with aiofiles.open("data/meta.txt", "r") as fs:
+        num = int(await fs.read()) + 1
+    async with aiofiles.open("data/meta.txt", "w") as fs:
+        await fs.write(str(num))
+    return num

FileStorage/views/__init__.py

@@ -0,0 +1,3 @@
+from .get_file import get_file
+from .upload_file import upload_file
+from .delete_file import delete_file

FileStorage/views/delete_file.py

@@ -0,0 +1,8 @@
+from os import remove
+
+from aiohttp import web
+
+
+async def delete_file(request):
+    remove("data/" + request.rel_url.query['id'])
+    return web.json_response({"success": True})

FileStorage/views/get_file.py

@@ -0,0 +1,10 @@
+import aiofiles
+from aiohttp import web
+
+
+async def get_file(request):
+    response = web.StreamResponse()
+    await response.prepare(request)
+    async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs:
+        await response.write_eof(await fs.read())
+    return response

FileStorage/views/upload_file.py

@@ -0,0 +1,11 @@
+from aiohttp import web
+
+from FileStorage.sync import write_meta
+import aiofiles
+
+
+async def upload_file(request):
+    file_id = await write_meta(request)
+    async with aiofiles.open("data/" + str(file_id), "wb") as fs:
+        await fs.write(await request.content.read())
+    return web.json_response({"id": file_id})

Main/management/commands/storage.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+from FileStorage.root import runserver
+
+
+class Command(BaseCommand):
+    help = 'starts FileStorage'
+
+    def handle(self, *args, **options):
+        runserver()

Main/migrations/0075_auto_20211110_2317.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.4 on 2021-11-10 20:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('Main', '0074_auto_20211106_1215'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SolutionFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.TextField()),
+                ('fs_id', models.IntegerField()),
+                ('solution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='Main.solution')),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='Language',
+        ),
+        migrations.AddField(
+            model_name='extrafile',
+            name='fs_id',
+            field=models.IntegerField(null=True),
+        ),
+    ]

Main/models/__init__.py

@@ -8,3 +8,4 @@ from Main.models.settask import SetTask
 from Main.models.solution import Solution
 from Main.models.extrafile import ExtraFile
 from Main.models.progress import Progress
+from Main.models.solution_file import SolutionFile

Main/models/extrafile.py

@@ -1,23 +1,17 @@
-from os import remove
-from os.path import join, exists
-
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 
-from Sprint.settings import DATA_ROOT
+from .mixins import FileStorageMixin
 
 
-class ExtraFile(models.Model):
+class ExtraFile(FileStorageMixin, models.Model):
     task = models.ForeignKey("Task", on_delete=models.CASCADE)
     filename = models.TextField()
     is_test = models.BooleanField(null=True)
     is_sample = models.BooleanField(null=True)
     readable = models.BooleanField(null=True)
     test_number = models.IntegerField(null=True)
-
-    @property
-    def path(self):
-        return join(DATA_ROOT, "extra_files", str(self.id))
+    fs_id = models.IntegerField(null=True)
 
     @property
     def can_be_sample(self):
@@ -29,13 +23,8 @@ class ExtraFile(models.Model):
             )
         )
 
-    @property
-    def text(self):
-        return open(self.path, "r").read()
-
     def delete(self, using=None, keep_parents=False):
-        if exists(self.path):
-            remove(self.path)
+        self.remove_from_fs()
         if self.is_test and self.filename.endswith('.a'):
             try:
                 ef = ExtraFile.objects.get(task=self.task, filename=self.filename.rstrip('.a'), is_test=True)
@@ -47,4 +36,4 @@ class ExtraFile(models.Model):
 
     @property
     def answer(self):
-        return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a')
+        return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a')

Main/models/mixins.py

@@ -0,0 +1,14 @@
+from SprintLib.utils import get_bytes, write_bytes, delete_file
+
+
+class FileStorageMixin:
+    @property
+    def text(self):
+        return get_bytes(self.fs_id).decode("utf-8")
+
+    def write(self, bytes):
+        self.fs_id = write_bytes(bytes)
+        self.save()
+
+    def remove_from_fs(self):
+        delete_file(self.fs_id)

Main/models/solution.py

@@ -1,13 +1,12 @@
-from os import mkdir, walk
 from os.path import join, exists
 from shutil import rmtree
 from subprocess import call
 
 from django.contrib.auth.models import User
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils import timezone
 
+from Main.models.solution_file import SolutionFile
 from Main.models.task import Task
 from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL
 from SprintLib.language import languages
@@ -32,47 +31,41 @@ class Solution(models.Model):
     @property
     def files(self):
         data = []
-        for path, _, files in walk(self.directory):
-            if path.startswith(self.testing_directory):
+        for file in SolutionFile.objects.filter(solution=self):
+            try:
+                text = file.text
+            except:
                 continue
-            for file in files:
-                try:
-                    entity = {
-                        'filename': file,
-                        'text': open(join(path, file), 'r').read()
-                    }
-                    end = file.split('.')[-1]
-                    language = None
-                    for l in languages:
-                        if l.file_type == end:
-                            language = l
-                            break
-                    if language is None:
-                        highlight = 'nohighlight'
-                    else:
-                        highlight = 'language-' + language.highlight
-                    entity['highlight'] = highlight
-                    data.append(entity)
-                except:
-                    continue
+            entity = {
+                'filename': file.path,
+                'text': text
+            }
+            end = file.path.split('.')[-1]
+            language = None
+            for l in languages:
+                if l.file_type == end:
+                    language = l
+                    break
+            if language is None:
+                highlight = 'nohighlight'
+            else:
+                highlight = 'language-' + language.highlight
+            entity['highlight'] = highlight
+            data.append(entity)
         data.sort(key=lambda x: x['filename'])
         return data
 
-    def create_dirs(self):
-        mkdir(self.directory)
-        mkdir(self.testing_directory)
-
     @property
     def directory(self):
-        return join(SOLUTIONS_ROOT, str(self.id))
+        return "solutions/" + str(self.id)
 
     @property
     def testing_directory(self):
-        return join(self.directory, 'test_dir')
+        return self.directory
 
     @property
     def volume_directory(self):
-        return join(SOLUTIONS_ROOT_EXTERNAL, str(self.id), 'test_dir')
+        return "/sprint-data/worker/" + str(self.id)
 
     def exec_command(self, command, working_directory='app', timeout=None):
         return call(f'docker exec -i solution_{self.id} sh -c "cd {working_directory} && {command}"', shell=True, timeout=timeout)

Main/models/solution_file.py

@@ -0,0 +1,9 @@
+from django.db import models
+
+from Main.models.mixins import FileStorageMixin
+
+
+class SolutionFile(FileStorageMixin, models.Model):
+    path = models.TextField()
+    fs_id = models.IntegerField()
+    solution = models.ForeignKey('Solution', on_delete=models.CASCADE)

Main/views/TaskSettingsView.py

@@ -45,11 +45,9 @@ class TaskSettingsView(BaseView):
                 filename=filename,
                 is_test=is_test
             )
-        with open(ef.path, 'wb') as fs:
-            for chunk in self.request.FILES['file'].chunks():
-                fs.write(chunk)
+        ef.write(self.request.FILES['file'].read())
         try:
-            open(ef.path, 'r').read()
+            var = ef.text
             ef.readable = True
         except UnicodeDecodeError:
             ef.readable = False
@@ -73,8 +71,7 @@ class TaskSettingsView(BaseView):
         ef, created = ExtraFile.objects.get_or_create(filename=name, task=self.entities.task)
         if not created:
             return f'/admin/task?task_id={self.entities.task.id}&error_message=Файл с таким именем уже существует'
-        with open(ef.path, 'w') as fs:
-            fs.write('')
+        ef.write(b"")
         ef.is_test = is_test
         ef.readable = True
         ef.save()
@@ -88,8 +85,8 @@ class TaskSettingsView(BaseView):
 
     def post_save_test(self):
         ef = ExtraFile.objects.get(id=self.request.POST['test_id'])
-        with open(ef.path, 'w') as fs:
-            fs.write(self.request.POST['text'])
+        ef.remove_from_fs()
+        ef.write(self.request.POST['text'].encode('utf-8'))
         ef.is_sample = 'is_sample' in self.request.POST.keys()
         ef.save()
         return f'/admin/task?task_id={self.entities.task.id}'

Main/views/TaskView.py

@@ -1,11 +1,11 @@
+import io
 from zipfile import ZipFile
-from os.path import join
 
-from Main.models import Solution, Progress
+from Main.models import Solution, Progress, SolutionFile
 from SprintLib.BaseView import BaseView
 from SprintLib.language import languages
 from SprintLib.queue import send_testing
-from SprintLib.testers import *
+from SprintLib.utils import write_bytes
 
 
 class TaskView(BaseView):
@@ -13,40 +13,54 @@ class TaskView(BaseView):
     view_file = "task.html"
 
     def get(self):
-        self.context['languages'] = sorted(languages, key=lambda x: x.name)
-        progress, _ = Progress.objects.get_or_create(user=self.request.user, task=self.entities.task)
-        self.context['progress'] = progress
+        self.context["languages"] = sorted(languages, key=lambda x: x.name)
+        progress, _ = Progress.objects.get_or_create(
+            user=self.request.user, task=self.entities.task
+        )
+        self.context["progress"] = progress
 
     def pre_handle(self):
-        if self.request.method == 'GET':
+        if self.request.method == "GET":
             return
         self.solution = Solution.objects.create(
             task=self.entities.task,
             user=self.request.user,
-            language_id=int(self.request.POST["language"])
+            language_id=int(self.request.POST["language"]),
         )
-        self.solution.create_dirs()
 
     def post_0(self):
         # отправка решения через текст
-        filename = 'solution.' + self.solution.language.file_type
-        file_path = join(self.solution.directory, filename)
-        with open(file_path, 'w') as fs:
-            fs.write(self.request.POST['code'])
+        fs_id = write_bytes(self.request.POST["code"].encode("utf-8"))
+        SolutionFile.objects.create(
+            path="solution." + self.solution.language.file_type,
+            solution=self.solution,
+            fs_id=fs_id,
+        )
         send_testing(self.solution.id)
         return "task?task_id=" + str(self.entities.task.id)
 
     def post_1(self):
         # отправка решения через файл
-        if 'file' not in self.request.FILES:
+        if "file" not in self.request.FILES:
             return "task?task_id=" + str(self.entities.task.id)
-        filename = self.request.FILES['file'].name
-        file_path = join(self.solution.directory, filename)
-        with open(file_path, 'wb') as fs:
-            for chunk in self.request.FILES['file'].chunks():
-                fs.write(chunk)
-        if filename.endswith('.zip'):
-            with ZipFile(file_path) as obj:
-                obj.extractall(self.solution.directory)
+        filename = self.request.FILES["file"].name
+        if filename.endswith(".zip"):
+            archive = ZipFile(io.BytesIO(self.request.FILES['file'].read()))
+            for file in archive.infolist():
+                if file.is_dir():
+                    continue
+                fs_id = write_bytes(archive.read(file.filename))
+                SolutionFile.objects.create(
+                    path=file.filename,
+                    solution=self.solution,
+                    fs_id=fs_id,
+                )
+        else:
+            fs_id = write_bytes(self.request.FILES['file'].read())
+            SolutionFile.objects.create(
+                path=filename,
+                solution=self.solution,
+                fs_id=fs_id,
+            )
         send_testing(self.solution.id)
         return "task?task_id=" + str(self.entities.task.id)

Sprint/settings.py

@@ -146,6 +146,9 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions")
 RABBIT_HOST = HOST
 RABBIT_PORT = 5672
 
+FS_HOST = "http://" + HOST
+FS_PORT = 5555
+
 STATICFILES_DIRS = [
     os.path.join(BASE_DIR, "Main/static"),
 ]

SprintLib/testers/BaseTester.py

@@ -1,13 +1,12 @@
 from os import listdir, mkdir
 from os.path import join, exists
-from shutil import copyfile, rmtree
 from subprocess import call, TimeoutExpired
 
 from Main.management.commands.bot import bot
-from Main.models import ExtraFile
+from Main.models import ExtraFile, SolutionFile
 from Main.models.progress import Progress
 from Sprint.settings import CONSTS
-from SprintLib.utils import copy_content
+from SprintLib.utils import get_bytes
 
 
 class TestException(Exception):
@@ -56,19 +55,26 @@ class BaseTester:
         self.solution = solution
 
     def execute(self):
-        if not exists(self.solution.testing_directory):
-            mkdir(self.solution.testing_directory)
-        copy_content(
-            self.solution.directory, self.solution.testing_directory, ("test_dir",)
-        )
+        if not exists("solutions"):
+            mkdir("solutions")
+        mkdir("solutions/" + str(self.solution.id))
+        for file in SolutionFile.objects.filter(solution=self.solution):
+            dirs = file.path.split('/')
+            for i in range(len(dirs) - 1):
+                name = join(str("solutions/" + self.solution.id), '/'.join(dirs[:i + 1]))
+                if not exists(name):
+                    mkdir(name)
+            with open(join("solutions/" + str(self.solution.id), file.path), 'wb') as fs:
+                fs.write(get_bytes(file.fs_id))
         self.solution.result = CONSTS["testing_status"]
         self.solution.save()
-        docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}"
+        docker_command = f"docker run --name solution_{self.solution.id} --volume=/sprint-data/solutions/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}"
         print(docker_command)
         call(docker_command, shell=True)
         print("Container created")
         for file in ExtraFile.objects.filter(task=self.solution.task):
-            copyfile(file.path, join(self.solution.testing_directory, file.filename))
+            with open(join("solutions/" + str(self.solution.id), file.filename), 'wb') as fs:
+                fs.write(get_bytes(file.fs_id))
         print("Files copied")
         try:
             self.before_test()
@@ -98,7 +104,6 @@ class BaseTester:
             print(str(e))
         self.solution.save()
         call(f"docker rm --force solution_{self.solution.id}", shell=True)
-        rmtree(self.solution.testing_directory)
         self.solution.user.userinfo.refresh_from_db()
         if self.solution.user.userinfo.notification_solution_result:
             bot.send_message(

SprintLib/testers/Python3Tester.py

@@ -1,4 +1,4 @@
-from os import listdir
+from os import listdir, getcwd
 
 from SprintLib.testers.BaseTester import BaseTester, TestException
 
@@ -7,11 +7,13 @@ class Python3Tester(BaseTester):
     file = None
 
     def before_test(self):
+        print(getcwd())
         for file in listdir(self.solution.testing_directory):
             if file.endswith(".py"):
                 self.file = file
                 break
         if self.file is None:
+            print('no file')
             raise TestException("TE")
 
     @property

SprintLib/utils.py

@@ -1,12 +1,21 @@
-from os import listdir
-from os.path import isfile, join
-from shutil import copyfile, copytree
+from requests import get, post
+
+from Sprint import settings
 
 
-def copy_content(from_dir, to_dir, exc=()):
-    for file in listdir(from_dir):
-        if file in exc:
-            continue
-        full_path = join(from_dir, file)
-        func = copyfile if isfile(full_path) else copytree
-        func(full_path, join(to_dir, file))
+def write_bytes(data):
+    url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file"
+    print(url)
+    return post(url, data=data).json()['id']
+
+
+def get_bytes(num):
+    url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num)
+    print(url)
+    return get(url).content
+
+
+def delete_file(num):
+    url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num)
+    print(url)
+    post(url)

docker-compose.yaml

@@ -27,11 +27,20 @@ services:
     ports:
       - "${PORT}:${PORT}"
     volumes:
-      - /sprint-data/data:/usr/src/app/data
       - /sprint-data/media:/usr/src/app/media
     depends_on:
       - postgres
       - rabbitmq
+      - storage
+
+  storage:
+    restart: always
+    image: mathwave/sprint-repo:sprint
+    command: python manage.py storage
+    ports:
+      - "5555:5555"
+    volumes:
+      - /sprint-data/data:/usr/src/app/data
 
   bot:
     image: mathwave/sprint-repo:sprint
@@ -63,6 +72,7 @@ services:
       - web
       - rabbitmq
       - postgres
+      - storage
     volumes:
-      - /sprint-data/data:/usr/src/app/data
+      - /sprint-data/solutions:/usr/src/app/solutions
       - /var/run/docker.sock:/var/run/docker.sock

requirements.txt

@@ -1,3 +1,5 @@
+aiofiles==0.7.0
+aiohttp==3.8.0
 amqp==5.0.6
 asgiref==3.3.4
 billiard==3.6.4.0

scripts/runserver.sh

@@ -1,3 +1,2 @@
 python manage.py migrate
-python manage.py update_languages
 python manage.py runserver 0.0.0.0:$PORT --noreload