From 80e08675265796991587cf76dde62176bd4c7b2c Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 15 Mar 2022 23:09:38 +0300 Subject: [PATCH] dump --- Main/migrations/0028_dump.py | 30 +++++++++ Main/migrations/0029_remove_dump_ready.py | 17 ++++++ Main/models/__init__.py | 1 + Main/models/dump.py | 25 ++++++++ Main/models/task.py | 5 ++ Main/views/TaskSettingsView.py | 8 ++- daemons/management/commands/file_generator.py | 61 ++++++++++++++++++- templates/task_settings.html | 25 +++++++- 8 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 Main/migrations/0028_dump.py create mode 100644 Main/migrations/0029_remove_dump_ready.py create mode 100644 Main/models/dump.py diff --git a/Main/migrations/0028_dump.py b/Main/migrations/0028_dump.py new file mode 100644 index 0000000..54e5c3b --- /dev/null +++ b/Main/migrations/0028_dump.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.4 on 2022-03-15 19:08 + +import Main.models.mixins +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('Main', '0027_task_allow_sharing'), + ] + + operations = [ + migrations.CreateModel( + name='Dump', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(default=django.utils.timezone.now)), + ('fs_id', models.IntegerField(null=True)), + ('ready', models.BooleanField(default=False)), + ('executor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('task', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='Main.task')), + ], + bases=(Main.models.mixins.FileStorageMixin, models.Model), + ), + ] diff --git a/Main/migrations/0029_remove_dump_ready.py b/Main/migrations/0029_remove_dump_ready.py new file mode 100644 index 0000000..61c3c02 --- /dev/null +++ b/Main/migrations/0029_remove_dump_ready.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2022-03-15 19:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('Main', '0028_dump'), + ] + + operations = [ + migrations.RemoveField( + model_name='dump', + name='ready', + ), + ] diff --git a/Main/models/__init__.py b/Main/models/__init__.py index 82152b8..bc4b339 100644 --- a/Main/models/__init__.py +++ b/Main/models/__init__.py @@ -10,3 +10,4 @@ from Main.models.progress import Progress from Main.models.solution_file import SolutionFile from Main.models.friendship import Friendship from Main.models.language_apply import LanguageApply +from Main.models.dump import Dump diff --git a/Main/models/dump.py b/Main/models/dump.py new file mode 100644 index 0000000..6a0a5e2 --- /dev/null +++ b/Main/models/dump.py @@ -0,0 +1,25 @@ +from django.contrib.auth.models import User +from django.db import models +from django.utils import timezone + +from .mixins import FileStorageMixin + + +class Dump(FileStorageMixin, models.Model): + timestamp = models.DateTimeField(default=timezone.now) + fs_id = models.IntegerField(null=True) + executor = models.ForeignKey(User, on_delete=models.CASCADE) + task = models.ForeignKey("Task", on_delete=models.CASCADE, null=True) + + @property + def ready(self): + return self.fs_id is not None + + @property + def str_date(self): + return self.timestamp.strftime("%Y-%m-%dT%H:%M:%S") + + @property + def filename(self): + if self.task is not None: + return f"dump-task-{self.task.id}-{self.str_date}" diff --git a/Main/models/task.py b/Main/models/task.py index b896fd1..0396945 100644 --- a/Main/models/task.py +++ b/Main/models/task.py @@ -2,6 +2,7 @@ from django.contrib.postgres.fields import ArrayField from django.db import models from django.contrib.auth.models import User +from Main.models.dump import Dump from Main.models.extrafile import ExtraFile @@ -21,6 +22,10 @@ class Task(models.Model): def __str__(self): return self.name + @property + def dumps(self): + return Dump.objects.filter(task=self).order_by("-timestamp") + @property def files(self): return ExtraFile.objects.filter(task=self, is_test=False) diff --git a/Main/views/TaskSettingsView.py b/Main/views/TaskSettingsView.py index 07ff59d..addb2c2 100644 --- a/Main/views/TaskSettingsView.py +++ b/Main/views/TaskSettingsView.py @@ -1,7 +1,8 @@ from django.http import HttpResponse -from Main.models import ExtraFile +from Main.models import ExtraFile, Dump from SprintLib.BaseView import BaseView, AccessError +from SprintLib.queue import send_to_queue class TaskSettingsView(BaseView): @@ -23,6 +24,11 @@ class TaskSettingsView(BaseView): self.entities.task.save() return f"/admin/task?task_id={self.entities.task.id}" + def post_dump(self): + dump = Dump.objects.create(executor=self.request.user, task=self.entities.task) + send_to_queue("files", {"id": dump.id}) + return f"/admin/task?task_id={self.entities.task.id}" + def _upload(self, is_test): filename = self.request.FILES["file"].name ef, created = None, None diff --git a/daemons/management/commands/file_generator.py b/daemons/management/commands/file_generator.py index a94b33b..b13271f 100644 --- a/daemons/management/commands/file_generator.py +++ b/daemons/management/commands/file_generator.py @@ -1,9 +1,68 @@ +import json +from os import mkdir, listdir, remove +from os.path import exists, join +from shutil import rmtree +from zipfile import ZipFile + +from Main.models import Dump, ExtraFile from SprintLib.queue import MessagingSupport +from SprintLib.utils import write_bytes class Command(MessagingSupport): help = "starts file generator" queue_name = "files" + def process_task(self, dump): + task_fields = [ + 'name', + 'public', + 'legend', + 'input_format', + 'output_format', + 'specifications', + 'time_limit', + 'time_estimation', + ] + task_data = {field: getattr(dump.task, field) for field in task_fields} + files_fields = [ + 'id', + 'filename', + 'is_test', + 'is_sample', + 'readable', + 'test_number' + ] + task_data['files'] = [ + { + field: getattr(file, field) + for field in files_fields + } + for file in ExtraFile.objects.filter(task=dump.task) + ] + tempdir = "/var/tmp/dump/" + dump_filename = 'dump.zip' + try: + mkdir(tempdir) + with open(join(tempdir, 'meta.json'), 'w') as fs: + json.dump(task_data, fs) + for ef in ExtraFile.objects.filter(task=dump.task): + with open(join(tempdir, ef.id), 'wb') as fs: + fs.write(ef.bytes) + with ZipFile(dump_filename, 'w') as zipfile: + for file in listdir(tempdir): + zipfile.write(join(tempdir, file)) + fs_id = write_bytes(open(dump_filename, 'rb').read()) + dump.fs_id = fs_id + dump.save() + finally: + if exists(tempdir): + rmtree(tempdir) + if exists(dump_filename): + remove(dump_filename) + def process(self, payload: dict): - ... + id = payload['id'] + dump = Dump.objects.get(id=id) + if dump.task: + self.process_task(dump) diff --git a/templates/task_settings.html b/templates/task_settings.html index 5ebd3a5..00e5db7 100644 --- a/templates/task_settings.html +++ b/templates/task_settings.html @@ -311,5 +311,28 @@ - +

+

Дампы

+ + + + + + + + + {% for dump in task.dumps %} + + + + + + + {% endfor %} + +
Id дампаВремя запросаИнициаторГотовность
{{ dump.id }}{{ dump.timestamp }}{{ dump.executor.username }}{% if dump.ready %}Скачать{% else %} В процессе{% endif %}
+
+ {% csrf_token %} + +
{% endblock %} \ No newline at end of file