filestorage

This commit is contained in:
Egor Matveev 2021-11-11 11:47:06 +03:00
parent a8111c45e9
commit edb58e23a3
22 changed files with 237 additions and 93 deletions

View File

@ -38,6 +38,7 @@ deploy-dev:
SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions"
DB_HOST: "postgres"
RABBIT_HOST: "rabbitmq"
FS_HOST: "storage"
deploy-prod:
extends:

0
FileStorage/__init__.py Normal file
View File

21
FileStorage/root.py Normal file
View File

@ -0,0 +1,21 @@
import os
from os import mkdir
from os.path import exists
from aiohttp import web
from FileStorage.routes import setup_routes
def runserver():
app = web.Application()
setup_routes(app)
if not exists("data"):
mkdir("data")
if not exists("data/meta.txt"):
with open("data/meta.txt", "w") as fs:
fs.write("0")
web.run_app(app, host=os.getenv("FS_HOST", "0.0.0.0"), port=5555)
if __name__ == "__main__":
runserver()

9
FileStorage/routes.py Normal file
View File

@ -0,0 +1,9 @@
from aiohttp import web
from FileStorage.views import get_file, upload_file, delete_file
def setup_routes(app: web.Application):
app.router.add_get("/get_file", get_file)
app.router.add_post("/upload_file", upload_file)
app.router.add_post("/delete_file", delete_file)

26
FileStorage/sync.py Normal file
View File

@ -0,0 +1,26 @@
import threading
import aiofiles
def synchronized_method(method):
outer_lock = threading.Lock()
lock_name = "__" + method.__name__ + "_lock" + "__"
def sync_method(self, *args, **kws):
with outer_lock:
if not hasattr(self, lock_name):
setattr(self, lock_name, threading.Lock())
lock = getattr(self, lock_name)
with lock:
return method(self, *args, **kws)
return sync_method
@synchronized_method
async def write_meta(request):
async with aiofiles.open("data/meta.txt", "r") as fs:
num = int(await fs.read()) + 1
async with aiofiles.open("data/meta.txt", "w") as fs:
await fs.write(str(num))
return num

View File

@ -0,0 +1,3 @@
from .get_file import get_file
from .upload_file import upload_file
from .delete_file import delete_file

View File

@ -0,0 +1,8 @@
from os import remove
from aiohttp import web
async def delete_file(request):
remove("data/" + request.rel_url.query['id'])
return web.json_response({"success": True})

View File

@ -0,0 +1,10 @@
import aiofiles
from aiohttp import web
async def get_file(request):
response = web.StreamResponse()
await response.prepare(request)
async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs:
await response.write_eof(await fs.read())
return response

View File

@ -0,0 +1,11 @@
from aiohttp import web
from FileStorage.sync import write_meta
import aiofiles
async def upload_file(request):
file_id = await write_meta(request)
async with aiofiles.open("data/" + str(file_id), "wb") as fs:
await fs.write(await request.content.read())
return web.json_response({"id": file_id})

View File

@ -0,0 +1,9 @@
from django.core.management.base import BaseCommand
from FileStorage.root import runserver
class Command(BaseCommand):
help = 'starts FileStorage'
def handle(self, *args, **options):
runserver()

View File

@ -8,3 +8,4 @@ from Main.models.settask import SetTask
from Main.models.solution import Solution
from Main.models.extrafile import ExtraFile
from Main.models.progress import Progress
from Main.models.solution_file import SolutionFile

View File

@ -1,23 +1,17 @@
from os import remove
from os.path import join, exists
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from Sprint.settings import DATA_ROOT
from .mixins import FileStorageMixin
class ExtraFile(models.Model):
class ExtraFile(FileStorageMixin, models.Model):
task = models.ForeignKey("Task", on_delete=models.CASCADE)
filename = models.TextField()
is_test = models.BooleanField(null=True)
is_sample = models.BooleanField(null=True)
readable = models.BooleanField(null=True)
test_number = models.IntegerField(null=True)
@property
def path(self):
return join(DATA_ROOT, "extra_files", str(self.id))
fs_id = models.IntegerField(null=True)
@property
def can_be_sample(self):
@ -29,13 +23,8 @@ class ExtraFile(models.Model):
)
)
@property
def text(self):
return open(self.path, "r").read()
def delete(self, using=None, keep_parents=False):
if exists(self.path):
remove(self.path)
self.remove_from_fs()
if self.is_test and self.filename.endswith('.a'):
try:
ef = ExtraFile.objects.get(task=self.task, filename=self.filename.rstrip('.a'), is_test=True)
@ -47,4 +36,4 @@ class ExtraFile(models.Model):
@property
def answer(self):
return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a')
return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a')

14
Main/models/mixins.py Normal file
View File

@ -0,0 +1,14 @@
from SprintLib.utils import get_bytes, write_bytes, delete_file
class FileStorageMixin:
@property
def text(self):
return get_bytes(self.fs_id).decode("utf-8")
def write(self, bytes):
self.fs_id = write_bytes(bytes)
self.save()
def remove_from_fs(self):
delete_file(self.fs_id)

View File

@ -1,13 +1,12 @@
from os import mkdir, walk
from os.path import join, exists
from shutil import rmtree
from subprocess import call
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.utils import timezone
from Main.models.solution_file import SolutionFile
from Main.models.task import Task
from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL
from SprintLib.language import languages
@ -32,36 +31,30 @@ class Solution(models.Model):
@property
def files(self):
data = []
for path, _, files in walk(self.directory):
if path.startswith(self.testing_directory):
for file in SolutionFile.objects.filter(solution=self):
try:
text = file.text
except:
continue
for file in files:
try:
entity = {
'filename': file,
'text': open(join(path, file), 'r').read()
}
end = file.split('.')[-1]
language = None
for l in languages:
if l.file_type == end:
language = l
break
if language is None:
highlight = 'nohighlight'
else:
highlight = 'language-' + language.highlight
entity['highlight'] = highlight
data.append(entity)
except:
continue
entity = {
'filename': file.path,
'text': text
}
end = file.path.split('.')[-1]
language = None
for l in languages:
if l.file_type == end:
language = l
break
if language is None:
highlight = 'nohighlight'
else:
highlight = 'language-' + language.highlight
entity['highlight'] = highlight
data.append(entity)
data.sort(key=lambda x: x['filename'])
return data
def create_dirs(self):
mkdir(self.directory)
mkdir(self.testing_directory)
@property
def directory(self):
return join(SOLUTIONS_ROOT, str(self.id))

View File

@ -0,0 +1,9 @@
from django.db import models
from Main.models.mixins import FileStorageMixin
class SolutionFile(FileStorageMixin, models.Model):
path = models.TextField()
fs_id = models.IntegerField()
solution = models.ForeignKey('Solution', on_delete=models.CASCADE)

View File

@ -45,11 +45,9 @@ class TaskSettingsView(BaseView):
filename=filename,
is_test=is_test
)
with open(ef.path, 'wb') as fs:
for chunk in self.request.FILES['file'].chunks():
fs.write(chunk)
ef.write(self.request.FILES['file'].read())
try:
open(ef.path, 'r').read()
var = ef.text
ef.readable = True
except UnicodeDecodeError:
ef.readable = False
@ -73,8 +71,7 @@ class TaskSettingsView(BaseView):
ef, created = ExtraFile.objects.get_or_create(filename=name, task=self.entities.task)
if not created:
return f'/admin/task?task_id={self.entities.task.id}&error_message=Файл с таким именем уже существует'
with open(ef.path, 'w') as fs:
fs.write('')
ef.write(b"")
ef.is_test = is_test
ef.readable = True
ef.save()
@ -88,8 +85,8 @@ class TaskSettingsView(BaseView):
def post_save_test(self):
ef = ExtraFile.objects.get(id=self.request.POST['test_id'])
with open(ef.path, 'w') as fs:
fs.write(self.request.POST['text'])
ef.remove_from_fs()
ef.write(self.request.POST['text'].encode('utf-8'))
ef.is_sample = 'is_sample' in self.request.POST.keys()
ef.save()
return f'/admin/task?task_id={self.entities.task.id}'

View File

@ -1,11 +1,11 @@
import io
from zipfile import ZipFile
from os.path import join
from Main.models import Solution, Progress
from Main.models import Solution, Progress, SolutionFile
from SprintLib.BaseView import BaseView
from SprintLib.language import languages
from SprintLib.queue import send_testing
from SprintLib.testers import *
from SprintLib.utils import write_bytes
class TaskView(BaseView):
@ -13,40 +13,54 @@ class TaskView(BaseView):
view_file = "task.html"
def get(self):
self.context['languages'] = sorted(languages, key=lambda x: x.name)
progress, _ = Progress.objects.get_or_create(user=self.request.user, task=self.entities.task)
self.context['progress'] = progress
self.context["languages"] = sorted(languages, key=lambda x: x.name)
progress, _ = Progress.objects.get_or_create(
user=self.request.user, task=self.entities.task
)
self.context["progress"] = progress
def pre_handle(self):
if self.request.method == 'GET':
if self.request.method == "GET":
return
self.solution = Solution.objects.create(
task=self.entities.task,
user=self.request.user,
language_id=int(self.request.POST["language"])
language_id=int(self.request.POST["language"]),
)
self.solution.create_dirs()
def post_0(self):
# отправка решения через текст
filename = 'solution.' + self.solution.language.file_type
file_path = join(self.solution.directory, filename)
with open(file_path, 'w') as fs:
fs.write(self.request.POST['code'])
fs_id = write_bytes(self.request.POST["code"].encode("utf-8"))
SolutionFile.objects.create(
path="solution." + self.solution.language.file_type,
solution=self.solution,
fs_id=fs_id,
)
send_testing(self.solution.id)
return "task?task_id=" + str(self.entities.task.id)
def post_1(self):
# отправка решения через файл
if 'file' not in self.request.FILES:
if "file" not in self.request.FILES:
return "task?task_id=" + str(self.entities.task.id)
filename = self.request.FILES['file'].name
file_path = join(self.solution.directory, filename)
with open(file_path, 'wb') as fs:
for chunk in self.request.FILES['file'].chunks():
fs.write(chunk)
if filename.endswith('.zip'):
with ZipFile(file_path) as obj:
obj.extractall(self.solution.directory)
filename = self.request.FILES["file"].name
if filename.endswith(".zip"):
archive = ZipFile(io.BytesIO(self.request.FILES['file'].read()))
for file in archive.infolist():
if file.is_dir():
continue
fs_id = write_bytes(archive.read(file.filename))
SolutionFile.objects.create(
path=file.filename,
solution=self.solution,
fs_id=fs_id,
)
else:
fs_id = write_bytes(self.request.FILES['file'].read())
SolutionFile.objects.create(
path=filename,
solution=self.solution,
fs_id=fs_id,
)
send_testing(self.solution.id)
return "task?task_id=" + str(self.entities.task.id)

View File

@ -143,6 +143,9 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions")
RABBIT_HOST = os.getenv("RABBIT_HOST", "0.0.0.0")
RABBIT_PORT = 5672
FS_HOST = os.getenv("FS_HOST", "http://0.0.0.0")
FS_PORT = 5555
STATICFILES_DIRS = [
os.path.join(BASE_DIR, "Main/static"),
]

View File

@ -4,10 +4,10 @@ from shutil import copyfile, rmtree
from subprocess import call, TimeoutExpired
from Main.management.commands.bot import bot
from Main.models import ExtraFile
from Main.models import ExtraFile, SolutionFile
from Main.models.progress import Progress
from Sprint.settings import CONSTS
from SprintLib.utils import copy_content
from SprintLib.utils import get_bytes
class TestException(Exception):
@ -56,11 +56,15 @@ class BaseTester:
self.solution = solution
def execute(self):
if not exists(self.solution.testing_directory):
mkdir(self.solution.testing_directory)
copy_content(
self.solution.directory, self.solution.testing_directory, ("test_dir",)
)
mkdir("solution")
for file in SolutionFile.objects.filter(solution=self.solution):
dirs = file.path.split('/')
for i in range(len(dirs) - 1):
name = join("solution", '/'.join(dirs[:i + 1]))
if not exists(name):
mkdir(name)
with open(file.path, 'wb') as fs:
fs.write(get_bytes(file.fs_id))
self.solution.result = CONSTS["testing_status"]
self.solution.save()
docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}"
@ -68,7 +72,8 @@ class BaseTester:
call(docker_command, shell=True)
print("Container created")
for file in ExtraFile.objects.filter(task=self.solution.task):
copyfile(file.path, join(self.solution.testing_directory, file.filename))
with open(join("solution", file.filename), 'wb') as fs:
fs.write(get_bytes(file.fs_id))
print("Files copied")
try:
self.before_test()
@ -98,7 +103,7 @@ class BaseTester:
print(str(e))
self.solution.save()
call(f"docker rm --force solution_{self.solution.id}", shell=True)
rmtree(self.solution.testing_directory)
rmtree("solution")
self.solution.user.userinfo.refresh_from_db()
if self.solution.user.userinfo.notification_solution_result:
bot.send_message(

View File

@ -1,12 +1,21 @@
from os import listdir
from os.path import isfile, join
from shutil import copyfile, copytree
from requests import get, post
from Sprint import settings
def copy_content(from_dir, to_dir, exc=()):
for file in listdir(from_dir):
if file in exc:
continue
full_path = join(from_dir, file)
func = copyfile if isfile(full_path) else copytree
func(full_path, join(to_dir, file))
def write_bytes(data):
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file"
print(url)
return post(url, data=data).json()['id']
def get_bytes(num):
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num)
print(url)
return get(url).content
def delete_file(num):
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num)
print(url)
post(url)

View File

@ -24,6 +24,7 @@ services:
PORT: $PORT
DB_HOST: $DB_HOST
RABBIT_HOST: $RABBIT_HOST
FS_HOST: $FS_HOST
command: scripts/runserver.sh
ports:
- "${PORT}:${PORT}"
@ -33,6 +34,15 @@ services:
depends_on:
- postgres
- rabbitmq
- storage
storage:
restart: always
image: mathwave/sprint-repo:sprint
ports:
- "5555:5555"
volumes:
- /sprint-data/data:/usr/src/app/FileStorage/data
bot:
image: mathwave/sprint-repo:sprint

View File

@ -1,3 +1,5 @@
aiofiles==0.7.0
aiohttp==3.8.0
amqp==5.0.6
asgiref==3.3.4
billiard==3.6.4.0