filestorage
This commit is contained in:
parent
a8111c45e9
commit
edb58e23a3
@ -38,6 +38,7 @@ deploy-dev:
|
||||
SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions"
|
||||
DB_HOST: "postgres"
|
||||
RABBIT_HOST: "rabbitmq"
|
||||
FS_HOST: "storage"
|
||||
|
||||
deploy-prod:
|
||||
extends:
|
||||
|
0
FileStorage/__init__.py
Normal file
0
FileStorage/__init__.py
Normal file
21
FileStorage/root.py
Normal file
21
FileStorage/root.py
Normal file
@ -0,0 +1,21 @@
|
||||
import os
|
||||
from os import mkdir
|
||||
from os.path import exists
|
||||
|
||||
from aiohttp import web
|
||||
from FileStorage.routes import setup_routes
|
||||
|
||||
|
||||
def runserver():
|
||||
app = web.Application()
|
||||
setup_routes(app)
|
||||
if not exists("data"):
|
||||
mkdir("data")
|
||||
if not exists("data/meta.txt"):
|
||||
with open("data/meta.txt", "w") as fs:
|
||||
fs.write("0")
|
||||
web.run_app(app, host=os.getenv("FS_HOST", "0.0.0.0"), port=5555)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
runserver()
|
9
FileStorage/routes.py
Normal file
9
FileStorage/routes.py
Normal file
@ -0,0 +1,9 @@
|
||||
from aiohttp import web
|
||||
|
||||
from FileStorage.views import get_file, upload_file, delete_file
|
||||
|
||||
|
||||
def setup_routes(app: web.Application):
|
||||
app.router.add_get("/get_file", get_file)
|
||||
app.router.add_post("/upload_file", upload_file)
|
||||
app.router.add_post("/delete_file", delete_file)
|
26
FileStorage/sync.py
Normal file
26
FileStorage/sync.py
Normal file
@ -0,0 +1,26 @@
|
||||
import threading
|
||||
|
||||
import aiofiles
|
||||
|
||||
|
||||
def synchronized_method(method):
|
||||
outer_lock = threading.Lock()
|
||||
lock_name = "__" + method.__name__ + "_lock" + "__"
|
||||
|
||||
def sync_method(self, *args, **kws):
|
||||
with outer_lock:
|
||||
if not hasattr(self, lock_name):
|
||||
setattr(self, lock_name, threading.Lock())
|
||||
lock = getattr(self, lock_name)
|
||||
with lock:
|
||||
return method(self, *args, **kws)
|
||||
return sync_method
|
||||
|
||||
|
||||
@synchronized_method
|
||||
async def write_meta(request):
|
||||
async with aiofiles.open("data/meta.txt", "r") as fs:
|
||||
num = int(await fs.read()) + 1
|
||||
async with aiofiles.open("data/meta.txt", "w") as fs:
|
||||
await fs.write(str(num))
|
||||
return num
|
3
FileStorage/views/__init__.py
Normal file
3
FileStorage/views/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
from .get_file import get_file
|
||||
from .upload_file import upload_file
|
||||
from .delete_file import delete_file
|
8
FileStorage/views/delete_file.py
Normal file
8
FileStorage/views/delete_file.py
Normal file
@ -0,0 +1,8 @@
|
||||
from os import remove
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
|
||||
async def delete_file(request):
|
||||
remove("data/" + request.rel_url.query['id'])
|
||||
return web.json_response({"success": True})
|
10
FileStorage/views/get_file.py
Normal file
10
FileStorage/views/get_file.py
Normal file
@ -0,0 +1,10 @@
|
||||
import aiofiles
|
||||
from aiohttp import web
|
||||
|
||||
|
||||
async def get_file(request):
|
||||
response = web.StreamResponse()
|
||||
await response.prepare(request)
|
||||
async with aiofiles.open("data/" + request.rel_url.query['id'], "rb") as fs:
|
||||
await response.write_eof(await fs.read())
|
||||
return response
|
11
FileStorage/views/upload_file.py
Normal file
11
FileStorage/views/upload_file.py
Normal file
@ -0,0 +1,11 @@
|
||||
from aiohttp import web
|
||||
|
||||
from FileStorage.sync import write_meta
|
||||
import aiofiles
|
||||
|
||||
|
||||
async def upload_file(request):
|
||||
file_id = await write_meta(request)
|
||||
async with aiofiles.open("data/" + str(file_id), "wb") as fs:
|
||||
await fs.write(await request.content.read())
|
||||
return web.json_response({"id": file_id})
|
9
Main/management/commands/storage.py
Normal file
9
Main/management/commands/storage.py
Normal file
@ -0,0 +1,9 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from FileStorage.root import runserver
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'starts FileStorage'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
runserver()
|
@ -8,3 +8,4 @@ from Main.models.settask import SetTask
|
||||
from Main.models.solution import Solution
|
||||
from Main.models.extrafile import ExtraFile
|
||||
from Main.models.progress import Progress
|
||||
from Main.models.solution_file import SolutionFile
|
||||
|
@ -1,23 +1,17 @@
|
||||
from os import remove
|
||||
from os.path import join, exists
|
||||
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import models
|
||||
|
||||
from Sprint.settings import DATA_ROOT
|
||||
from .mixins import FileStorageMixin
|
||||
|
||||
|
||||
class ExtraFile(models.Model):
|
||||
class ExtraFile(FileStorageMixin, models.Model):
|
||||
task = models.ForeignKey("Task", on_delete=models.CASCADE)
|
||||
filename = models.TextField()
|
||||
is_test = models.BooleanField(null=True)
|
||||
is_sample = models.BooleanField(null=True)
|
||||
readable = models.BooleanField(null=True)
|
||||
test_number = models.IntegerField(null=True)
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return join(DATA_ROOT, "extra_files", str(self.id))
|
||||
fs_id = models.IntegerField(null=True)
|
||||
|
||||
@property
|
||||
def can_be_sample(self):
|
||||
@ -29,13 +23,8 @@ class ExtraFile(models.Model):
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return open(self.path, "r").read()
|
||||
|
||||
def delete(self, using=None, keep_parents=False):
|
||||
if exists(self.path):
|
||||
remove(self.path)
|
||||
self.remove_from_fs()
|
||||
if self.is_test and self.filename.endswith('.a'):
|
||||
try:
|
||||
ef = ExtraFile.objects.get(task=self.task, filename=self.filename.rstrip('.a'), is_test=True)
|
||||
@ -47,4 +36,4 @@ class ExtraFile(models.Model):
|
||||
|
||||
@property
|
||||
def answer(self):
|
||||
return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a')
|
||||
return ExtraFile.objects.get(task=self.task, is_test=True, filename=self.filename + '.a')
|
||||
|
14
Main/models/mixins.py
Normal file
14
Main/models/mixins.py
Normal file
@ -0,0 +1,14 @@
|
||||
from SprintLib.utils import get_bytes, write_bytes, delete_file
|
||||
|
||||
|
||||
class FileStorageMixin:
|
||||
@property
|
||||
def text(self):
|
||||
return get_bytes(self.fs_id).decode("utf-8")
|
||||
|
||||
def write(self, bytes):
|
||||
self.fs_id = write_bytes(bytes)
|
||||
self.save()
|
||||
|
||||
def remove_from_fs(self):
|
||||
delete_file(self.fs_id)
|
@ -1,13 +1,12 @@
|
||||
from os import mkdir, walk
|
||||
from os.path import join, exists
|
||||
from shutil import rmtree
|
||||
from subprocess import call
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
|
||||
from Main.models.solution_file import SolutionFile
|
||||
from Main.models.task import Task
|
||||
from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL
|
||||
from SprintLib.language import languages
|
||||
@ -32,36 +31,30 @@ class Solution(models.Model):
|
||||
@property
|
||||
def files(self):
|
||||
data = []
|
||||
for path, _, files in walk(self.directory):
|
||||
if path.startswith(self.testing_directory):
|
||||
for file in SolutionFile.objects.filter(solution=self):
|
||||
try:
|
||||
text = file.text
|
||||
except:
|
||||
continue
|
||||
for file in files:
|
||||
try:
|
||||
entity = {
|
||||
'filename': file,
|
||||
'text': open(join(path, file), 'r').read()
|
||||
}
|
||||
end = file.split('.')[-1]
|
||||
language = None
|
||||
for l in languages:
|
||||
if l.file_type == end:
|
||||
language = l
|
||||
break
|
||||
if language is None:
|
||||
highlight = 'nohighlight'
|
||||
else:
|
||||
highlight = 'language-' + language.highlight
|
||||
entity['highlight'] = highlight
|
||||
data.append(entity)
|
||||
except:
|
||||
continue
|
||||
entity = {
|
||||
'filename': file.path,
|
||||
'text': text
|
||||
}
|
||||
end = file.path.split('.')[-1]
|
||||
language = None
|
||||
for l in languages:
|
||||
if l.file_type == end:
|
||||
language = l
|
||||
break
|
||||
if language is None:
|
||||
highlight = 'nohighlight'
|
||||
else:
|
||||
highlight = 'language-' + language.highlight
|
||||
entity['highlight'] = highlight
|
||||
data.append(entity)
|
||||
data.sort(key=lambda x: x['filename'])
|
||||
return data
|
||||
|
||||
def create_dirs(self):
|
||||
mkdir(self.directory)
|
||||
mkdir(self.testing_directory)
|
||||
|
||||
@property
|
||||
def directory(self):
|
||||
return join(SOLUTIONS_ROOT, str(self.id))
|
||||
|
9
Main/models/solution_file.py
Normal file
9
Main/models/solution_file.py
Normal file
@ -0,0 +1,9 @@
|
||||
from django.db import models
|
||||
|
||||
from Main.models.mixins import FileStorageMixin
|
||||
|
||||
|
||||
class SolutionFile(FileStorageMixin, models.Model):
|
||||
path = models.TextField()
|
||||
fs_id = models.IntegerField()
|
||||
solution = models.ForeignKey('Solution', on_delete=models.CASCADE)
|
@ -45,11 +45,9 @@ class TaskSettingsView(BaseView):
|
||||
filename=filename,
|
||||
is_test=is_test
|
||||
)
|
||||
with open(ef.path, 'wb') as fs:
|
||||
for chunk in self.request.FILES['file'].chunks():
|
||||
fs.write(chunk)
|
||||
ef.write(self.request.FILES['file'].read())
|
||||
try:
|
||||
open(ef.path, 'r').read()
|
||||
var = ef.text
|
||||
ef.readable = True
|
||||
except UnicodeDecodeError:
|
||||
ef.readable = False
|
||||
@ -73,8 +71,7 @@ class TaskSettingsView(BaseView):
|
||||
ef, created = ExtraFile.objects.get_or_create(filename=name, task=self.entities.task)
|
||||
if not created:
|
||||
return f'/admin/task?task_id={self.entities.task.id}&error_message=Файл с таким именем уже существует'
|
||||
with open(ef.path, 'w') as fs:
|
||||
fs.write('')
|
||||
ef.write(b"")
|
||||
ef.is_test = is_test
|
||||
ef.readable = True
|
||||
ef.save()
|
||||
@ -88,8 +85,8 @@ class TaskSettingsView(BaseView):
|
||||
|
||||
def post_save_test(self):
|
||||
ef = ExtraFile.objects.get(id=self.request.POST['test_id'])
|
||||
with open(ef.path, 'w') as fs:
|
||||
fs.write(self.request.POST['text'])
|
||||
ef.remove_from_fs()
|
||||
ef.write(self.request.POST['text'].encode('utf-8'))
|
||||
ef.is_sample = 'is_sample' in self.request.POST.keys()
|
||||
ef.save()
|
||||
return f'/admin/task?task_id={self.entities.task.id}'
|
||||
|
@ -1,11 +1,11 @@
|
||||
import io
|
||||
from zipfile import ZipFile
|
||||
from os.path import join
|
||||
|
||||
from Main.models import Solution, Progress
|
||||
from Main.models import Solution, Progress, SolutionFile
|
||||
from SprintLib.BaseView import BaseView
|
||||
from SprintLib.language import languages
|
||||
from SprintLib.queue import send_testing
|
||||
from SprintLib.testers import *
|
||||
from SprintLib.utils import write_bytes
|
||||
|
||||
|
||||
class TaskView(BaseView):
|
||||
@ -13,40 +13,54 @@ class TaskView(BaseView):
|
||||
view_file = "task.html"
|
||||
|
||||
def get(self):
|
||||
self.context['languages'] = sorted(languages, key=lambda x: x.name)
|
||||
progress, _ = Progress.objects.get_or_create(user=self.request.user, task=self.entities.task)
|
||||
self.context['progress'] = progress
|
||||
self.context["languages"] = sorted(languages, key=lambda x: x.name)
|
||||
progress, _ = Progress.objects.get_or_create(
|
||||
user=self.request.user, task=self.entities.task
|
||||
)
|
||||
self.context["progress"] = progress
|
||||
|
||||
def pre_handle(self):
|
||||
if self.request.method == 'GET':
|
||||
if self.request.method == "GET":
|
||||
return
|
||||
self.solution = Solution.objects.create(
|
||||
task=self.entities.task,
|
||||
user=self.request.user,
|
||||
language_id=int(self.request.POST["language"])
|
||||
language_id=int(self.request.POST["language"]),
|
||||
)
|
||||
self.solution.create_dirs()
|
||||
|
||||
def post_0(self):
|
||||
# отправка решения через текст
|
||||
filename = 'solution.' + self.solution.language.file_type
|
||||
file_path = join(self.solution.directory, filename)
|
||||
with open(file_path, 'w') as fs:
|
||||
fs.write(self.request.POST['code'])
|
||||
fs_id = write_bytes(self.request.POST["code"].encode("utf-8"))
|
||||
SolutionFile.objects.create(
|
||||
path="solution." + self.solution.language.file_type,
|
||||
solution=self.solution,
|
||||
fs_id=fs_id,
|
||||
)
|
||||
send_testing(self.solution.id)
|
||||
return "task?task_id=" + str(self.entities.task.id)
|
||||
|
||||
def post_1(self):
|
||||
# отправка решения через файл
|
||||
if 'file' not in self.request.FILES:
|
||||
if "file" not in self.request.FILES:
|
||||
return "task?task_id=" + str(self.entities.task.id)
|
||||
filename = self.request.FILES['file'].name
|
||||
file_path = join(self.solution.directory, filename)
|
||||
with open(file_path, 'wb') as fs:
|
||||
for chunk in self.request.FILES['file'].chunks():
|
||||
fs.write(chunk)
|
||||
if filename.endswith('.zip'):
|
||||
with ZipFile(file_path) as obj:
|
||||
obj.extractall(self.solution.directory)
|
||||
filename = self.request.FILES["file"].name
|
||||
if filename.endswith(".zip"):
|
||||
archive = ZipFile(io.BytesIO(self.request.FILES['file'].read()))
|
||||
for file in archive.infolist():
|
||||
if file.is_dir():
|
||||
continue
|
||||
fs_id = write_bytes(archive.read(file.filename))
|
||||
SolutionFile.objects.create(
|
||||
path=file.filename,
|
||||
solution=self.solution,
|
||||
fs_id=fs_id,
|
||||
)
|
||||
else:
|
||||
fs_id = write_bytes(self.request.FILES['file'].read())
|
||||
SolutionFile.objects.create(
|
||||
path=filename,
|
||||
solution=self.solution,
|
||||
fs_id=fs_id,
|
||||
)
|
||||
send_testing(self.solution.id)
|
||||
return "task?task_id=" + str(self.entities.task.id)
|
||||
|
@ -143,6 +143,9 @@ SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions")
|
||||
RABBIT_HOST = os.getenv("RABBIT_HOST", "0.0.0.0")
|
||||
RABBIT_PORT = 5672
|
||||
|
||||
FS_HOST = os.getenv("FS_HOST", "http://0.0.0.0")
|
||||
FS_PORT = 5555
|
||||
|
||||
STATICFILES_DIRS = [
|
||||
os.path.join(BASE_DIR, "Main/static"),
|
||||
]
|
||||
|
@ -4,10 +4,10 @@ from shutil import copyfile, rmtree
|
||||
from subprocess import call, TimeoutExpired
|
||||
|
||||
from Main.management.commands.bot import bot
|
||||
from Main.models import ExtraFile
|
||||
from Main.models import ExtraFile, SolutionFile
|
||||
from Main.models.progress import Progress
|
||||
from Sprint.settings import CONSTS
|
||||
from SprintLib.utils import copy_content
|
||||
from SprintLib.utils import get_bytes
|
||||
|
||||
|
||||
class TestException(Exception):
|
||||
@ -56,11 +56,15 @@ class BaseTester:
|
||||
self.solution = solution
|
||||
|
||||
def execute(self):
|
||||
if not exists(self.solution.testing_directory):
|
||||
mkdir(self.solution.testing_directory)
|
||||
copy_content(
|
||||
self.solution.directory, self.solution.testing_directory, ("test_dir",)
|
||||
)
|
||||
mkdir("solution")
|
||||
for file in SolutionFile.objects.filter(solution=self.solution):
|
||||
dirs = file.path.split('/')
|
||||
for i in range(len(dirs) - 1):
|
||||
name = join("solution", '/'.join(dirs[:i + 1]))
|
||||
if not exists(name):
|
||||
mkdir(name)
|
||||
with open(file.path, 'wb') as fs:
|
||||
fs.write(get_bytes(file.fs_id))
|
||||
self.solution.result = CONSTS["testing_status"]
|
||||
self.solution.save()
|
||||
docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}"
|
||||
@ -68,7 +72,8 @@ class BaseTester:
|
||||
call(docker_command, shell=True)
|
||||
print("Container created")
|
||||
for file in ExtraFile.objects.filter(task=self.solution.task):
|
||||
copyfile(file.path, join(self.solution.testing_directory, file.filename))
|
||||
with open(join("solution", file.filename), 'wb') as fs:
|
||||
fs.write(get_bytes(file.fs_id))
|
||||
print("Files copied")
|
||||
try:
|
||||
self.before_test()
|
||||
@ -98,7 +103,7 @@ class BaseTester:
|
||||
print(str(e))
|
||||
self.solution.save()
|
||||
call(f"docker rm --force solution_{self.solution.id}", shell=True)
|
||||
rmtree(self.solution.testing_directory)
|
||||
rmtree("solution")
|
||||
self.solution.user.userinfo.refresh_from_db()
|
||||
if self.solution.user.userinfo.notification_solution_result:
|
||||
bot.send_message(
|
||||
|
@ -1,12 +1,21 @@
|
||||
from os import listdir
|
||||
from os.path import isfile, join
|
||||
from shutil import copyfile, copytree
|
||||
from requests import get, post
|
||||
|
||||
from Sprint import settings
|
||||
|
||||
|
||||
def copy_content(from_dir, to_dir, exc=()):
|
||||
for file in listdir(from_dir):
|
||||
if file in exc:
|
||||
continue
|
||||
full_path = join(from_dir, file)
|
||||
func = copyfile if isfile(full_path) else copytree
|
||||
func(full_path, join(to_dir, file))
|
||||
def write_bytes(data):
|
||||
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/upload_file"
|
||||
print(url)
|
||||
return post(url, data=data).json()['id']
|
||||
|
||||
|
||||
def get_bytes(num):
|
||||
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/get_file?id=" + str(num)
|
||||
print(url)
|
||||
return get(url).content
|
||||
|
||||
|
||||
def delete_file(num):
|
||||
url = settings.FS_HOST + ":" + str(settings.FS_PORT) + "/delete_file?id=" + str(num)
|
||||
print(url)
|
||||
post(url)
|
||||
|
@ -24,6 +24,7 @@ services:
|
||||
PORT: $PORT
|
||||
DB_HOST: $DB_HOST
|
||||
RABBIT_HOST: $RABBIT_HOST
|
||||
FS_HOST: $FS_HOST
|
||||
command: scripts/runserver.sh
|
||||
ports:
|
||||
- "${PORT}:${PORT}"
|
||||
@ -33,6 +34,15 @@ services:
|
||||
depends_on:
|
||||
- postgres
|
||||
- rabbitmq
|
||||
- storage
|
||||
|
||||
storage:
|
||||
restart: always
|
||||
image: mathwave/sprint-repo:sprint
|
||||
ports:
|
||||
- "5555:5555"
|
||||
volumes:
|
||||
- /sprint-data/data:/usr/src/app/FileStorage/data
|
||||
|
||||
bot:
|
||||
image: mathwave/sprint-repo:sprint
|
||||
|
@ -1,3 +1,5 @@
|
||||
aiofiles==0.7.0
|
||||
aiohttp==3.8.0
|
||||
amqp==5.0.6
|
||||
asgiref==3.3.4
|
||||
billiard==3.6.4.0
|
||||
|
Loading…
Reference in New Issue
Block a user