self/sprint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tester in docker works

Browse Source
This commit is contained in:
Egor Matveev 2021-11-02 23:33:38 +03:00
parent a45d3e104b
commit 928cc4489d
35 changed files with 321 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Dockerfile
View File

@ -1,13 +1,18 @@
FROM python:3.7
FROM docker:dind
RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN apk update && apk add postgresql-dev gcc python3-dev musl-dev jpeg-dev zlib-dev libjpeg
RUN pip3 install --no-cache --upgrade pip setuptools
RUN addgroup -S docker
ENV PYTHONUNBUFFERED 1
ENV DJANGO_SETTINGS_MODULE Sprint.settings
RUN mkdir -p /usr/src/app/
WORKDIR /usr/src/app/
COPY . /usr/src/app/
COPY ../.. /usr/src/app/
RUN pip install --upgrade pip wheel setuptools
RUN pip install -r requirements.txt
RUN pip3 install -r requirements.txt
EXPOSE 8000

0
daemons/__init__.py → Main/management/__init__.py
View File

0
Main/management/commands/__init__.py Normal file
View File

32
Main/management/commands/receive.py Normal file
View File

@ -0,0 +1,32 @@
import pika
from django.core.management.base import BaseCommand
from Main.models import Solution
from Sprint import settings
from SprintLib.testers import *
class Command(BaseCommand):
help = 'Tests solution'
def handle(self, *args, **options):
print("Enter worker")
connection = pika.BlockingConnection(pika.ConnectionParameters(host=settings.RABBIT_HOST))
channel = connection.channel()
channel.queue_declare(queue='test')
def callback(ch, method, properties, body):
try:
id = int(str(body, encoding='utf-8'))
print(f"Received id {id}")
solution = Solution.objects.get(id=id)
except:
return
try:
eval(solution.language.work_name + 'Tester')(solution).execute()
except:
solution.result = 'TE'
solution.save()
channel.basic_consume(queue='test', on_message_callback=callback, auto_ack=True)
channel.start_consuming()

15
Main/management/commands/test.py Normal file
View File

@ -0,0 +1,15 @@
from django.core.management.base import BaseCommand
from Main.models import Solution
from SprintLib.testers import *
class Command(BaseCommand):
help = 'Tests solution'
def add_arguments(self, parser):
parser.add_argument('solution_id', type=int)
def handle(self, *args, **options):
solution = Solution.objects.get(id=options['solution_id'])
eval(solution.language.work_name + 'Tester')(solution).execute()

8
Main/management/commands/update_languages.py Normal file
View File

@ -0,0 +1,8 @@
from django.core.management.base import BaseCommand
class Command(BaseCommand):
help = 'Updates languages'
def handle(self, *args, **options):
pass

8
Main/models/solution.py
View File

@ -10,7 +10,7 @@ from django.utils import timezone
from Main.models.task import Task
from Main.models.language import Language
from Sprint.settings import CONSTS, SOLUTIONS_ROOT
from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL
class Solution(models.Model):
@ -61,5 +61,9 @@ class Solution(models.Model):
def testing_directory(self):
return join(self.directory, 'test_dir')
@property
def volume_directory(self):
return join(SOLUTIONS_ROOT_EXTERNAL, str(self.id), 'test_dir')
def exec_command(self, command, working_directory='app', timeout=None):
return call(f'docker exec -i solution_{self.id} bash -c "cd {working_directory} && {command}"', shell=True, timeout=timeout)
return call(f'docker exec -i solution_{self.id} sh -c "cd {working_directory} && {command}"', shell=True, timeout=timeout)

9
Main/tasks.py
View File

@ -1,9 +0,0 @@
from Main.models import Solution
from Sprint.celery import app
from SprintLib.testers import *
@app.task
def start_testing(solution_id):
solution = Solution.objects.get(id=solution_id)
eval(solution.language.work_name + 'Tester')(solution).execute()

10
Main/views/TaskView.py
View File

@ -1,8 +1,8 @@
from zipfile import ZipFile
from Main.models import Solution, Progress
from Main.tasks import start_testing
from Main.models import Solution
from SprintLib.BaseView import BaseView, Language
from SprintLib.queue import send_testing
from SprintLib.testers import *
@ -31,11 +31,13 @@ class TaskView(BaseView):
file_path = join(self.solution.directory, filename)
with open(file_path, 'w') as fs:
fs.write(self.request.POST['code'])
start_testing.delay(self.solution.id)
send_testing(self.solution.id)
return "task?task_id=" + str(self.entities.task.id)
def post_1(self):
# отправка решения через файл
if 'file' not in self.request.FILES:
return "task?task_id=" + str(self.entities.task.id)
filename = self.request.FILES['file'].name
file_path = join(self.solution.directory, filename)
with open(file_path, 'wb') as fs:
@ -44,5 +46,5 @@ class TaskView(BaseView):
if filename.endswith('.zip'):
with ZipFile(file_path) as obj:
obj.extractall(self.solution.directory)
start_testing.delay(self.solution.id)
send_testing(self.solution.id)
return "task?task_id=" + str(self.entities.task.id)

3
Sprint/__init__.py
View File

@ -1,3 +0,0 @@
from .celery import app as celery
__all__ = ('celery',)

17
Sprint/celery.py
View File

@ -1,17 +0,0 @@
import os
from celery import Celery
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Sprint.settings')
app = Celery('Sprint')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django apps.
app.autodiscover_tasks()

18
Sprint/settings.py
View File

@ -84,11 +84,11 @@ WSGI_APPLICATION = "Sprint.wsgi.application"
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql_psycopg2",
"NAME": "postgres",
"NAME": "sprint",
"USER": "postgres",
"PASSWORD": "password",
"HOST": "0.0.0.0",
"PORT": "5432",
"HOST": "postgres",
"PORT": 5432,
}
}
@ -141,6 +141,12 @@ MEDIA_ROOT = os.path.join(BASE_DIR, "media")
DATA_ROOT = os.path.join(BASE_DIR, "data")
SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions")
SOLUTIONS_ROOT_EXTERNAL = "/Users/egormatveev/Sprint/data/solutions"
RABBIT_HOST = "rabbitmq"
RABBIT_PORT = 5672
STATICFILES_DIRS = [
os.path.join(BASE_DIR, "Main/static"),
]
@ -149,12 +155,6 @@ STATICFILES_DIRS = [
# Authentication backends
AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",)
# Celery Configuration Options
CELERY_TIMEZONE = "Europe/Moscow"
CELERY_TASK_TRACK_STARTED = True
CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0'
CONSTS = {
"online_status": "Online",
"in_queue_status": "In queue",

17
SprintLib/BaseDaemon.py
View File

@ -1,17 +0,0 @@
import asyncio
import sys
class BaseDaemon:
def command(self):
raise NotImplementedError()
async def execute(self):
cmd = self.command()
proc = await asyncio.create_subprocess_shell(cmd, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
stdout, stderr = await proc.communicate()
print(f"[{cmd!r} exited with {proc.returncode}]")
if stdout:
print(f"[stdout]\n{stdout.decode()}")
if stderr:
print(f"[stderr]\n{stderr.decode()}")

12
SprintLib/queue.py Normal file
View File

@ -0,0 +1,12 @@
import pika
from Sprint import settings
def send_testing(solution_id):
with pika.BlockingConnection(
pika.ConnectionParameters(host=settings.RABBIT_HOST, port=settings.RABBIT_PORT)
) as connection:
channel = connection.channel()
channel.queue_declare(queue="test")
channel.basic_publish(exchange="", routing_key="test", body=bytes(str(solution_id), encoding='utf-8'))

55
SprintLib/testers/BaseTester.py
View File

@ -1,5 +1,5 @@
from os import listdir
from os.path import join
from os import listdir, mkdir
from os.path import join, exists
from shutil import copyfile, rmtree
from subprocess import call, TimeoutExpired
@ -18,10 +18,17 @@ class BaseTester:
working_directory = "app"
def before_test(self):
files = [file for file in listdir(self.solution.testing_directory) if file.endswith('.' + self.solution.language.file_type)]
code = self.solution.exec_command(f'{self.build_command} {" ".join(files)}', working_directory=self.working_directory)
files = [
file
for file in listdir(self.solution.testing_directory)
if file.endswith("." + self.solution.language.file_type)
]
code = self.solution.exec_command(
f'{self.build_command} {" ".join(files)}',
working_directory=self.working_directory,
)
if code != 0:
raise TestException('CE')
raise TestException("CE")
def test(self, filename):
code = self.solution.exec_command(
@ -30,9 +37,7 @@ class BaseTester:
)
if code != 0:
raise TestException("RE")
result = open(
join(self.solution.testing_directory, "output.txt"), "r"
).read()
result = open(join(self.solution.testing_directory, "output.txt"), "r").read()
if result.strip() != self.predicted.strip():
raise TestException("WA")
@ -51,17 +56,23 @@ class BaseTester:
self.solution = solution
def execute(self):
copy_content(self.solution.directory, self.solution.testing_directory, ('test_dir',))
if not exists(self.solution.testing_directory):
mkdir(self.solution.testing_directory)
copy_content(
self.solution.directory, self.solution.testing_directory, ("test_dir",)
)
self.solution.result = CONSTS["testing_status"]
self.solution.save()
call(
f"docker run --name solution_{self.solution.id} --volume={self.solution.testing_directory}:/{self.working_directory} -t -d {self.solution.language.image}",
shell=True,
)
docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}"
print(docker_command)
call(docker_command, shell=True)
print("Container created")
for file in ExtraFile.objects.filter(task=self.solution.task):
copyfile(file.path, join(self.solution.testing_directory, file.filename))
print("Files copied")
try:
self.before_test()
print("before test finished")
for test in self.solution.task.tests:
if not test.filename.endswith(".a"):
self.predicted = ExtraFile.objects.get(
@ -70,7 +81,9 @@ class BaseTester:
self.test(test.filename)
self.after_test()
self.solution.result = CONSTS["ok_status"]
progress = Progress.objects.get(user=self.solution.user, task=self.solution.task)
progress = Progress.objects.get(
user=self.solution.user, task=self.solution.task
)
if progress.finished_time is None:
progress.finished_time = self.solution.time_sent
progress.finished = True
@ -88,9 +101,11 @@ class BaseTester:
rmtree(self.solution.testing_directory)
self.solution.user.userinfo.refresh_from_db()
if self.solution.user.userinfo.notification_solution_result:
bot.send_message(self.solution.user.userinfo.telegram_chat_id,
f'Задача: {self.solution.task.name}\n'
f'Результат: {self.solution.result}\n'
f'Очки решения: {Progress.by_solution(self.solution).score}\n'
f'Текущий рейтинг: {self.solution.user.userinfo.rating}',
parse_mode='html')
bot.send_message(
self.solution.user.userinfo.telegram_chat_id,
f"Задача: {self.solution.task.name}\n"
f"Результат: {self.solution.result}\n"
f"Очки решения: {Progress.by_solution(self.solution).score}\n"
f"Текущий рейтинг: {self.solution.user.userinfo.rating}",
parse_mode="html",
)

2
SprintLib/testers/Python3Tester.py
View File

@ -16,4 +16,4 @@ class Python3Tester(BaseTester):
@property
def command(self):
return f"python {self.file}"
return f"python3 {self.file}"

3
apply_models.sh
View File

@ -1,3 +0,0 @@
source venv/bin/activate
python manage.py makemigrations
python manage.py migrate

6
daemons/bot.py
View File

@ -1,6 +0,0 @@
from SprintLib.BaseDaemon import BaseDaemon
class Daemon(BaseDaemon):
def command(self):
return "python bot.py"

6
daemons/celery.py
View File

@ -1,6 +0,0 @@
from SprintLib.BaseDaemon import BaseDaemon
class Daemon(BaseDaemon):
def command(self):
return "celery -A Sprint worker -l INFO --concurrency=4"

6
daemons/redis.py
View File

@ -1,6 +0,0 @@
from SprintLib.BaseDaemon import BaseDaemon
class Daemon(BaseDaemon):
def command(self):
return "redis-server"

6
daemons/web.py
View File

@ -1,6 +0,0 @@
from SprintLib.BaseDaemon import BaseDaemon
class Daemon(BaseDaemon):
def command(self):
return "python manage.py runserver 0.0.0.0:80"

50
docker-compose.yaml
View File

@ -5,11 +5,57 @@ services:
postgres:
restart: always
image: postgres
build:
context: .
dockerfile: dockerfiles/postgres/Dockerfile
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
POSTGRES_DB: postgres
volumes:
- ./postgres-data:/var/lib/postgresql/data
ports:
- "5432:5432"
web:
image: sprint
restart: always
command: scripts/runserver.sh
ports:
- "8000:8000"
volumes:
- .:/usr/src/app
depends_on:
- postgres
- rabbitmq
bot:
image: sprint
restart: always
command: python bot.py
depends_on:
- web
rabbitmq:
restart: always
build:
context: .
dockerfile: dockerfiles/rabbitmq/Dockerfile
ports:
- "15672:15672"
- "5672:5672"
worker:
restart: always
build:
context: .
dockerfile: dockerfiles/worker/Dockerfile
privileged: true
command: scripts/create_worker.sh
depends_on:
- web
- rabbitmq
- postgres
volumes:
- .:/usr/src/app
- /var/run/docker.sock:/var/run/docker.sock

2
dockerfiles/postgres/Dockerfile Normal file
View File

@ -0,0 +1,2 @@
FROM postgres
COPY ../../pg_hba.conf /var/lib/postgresql/data/pg_hba.conf

2
dockerfiles/rabbitmq/Dockerfile Normal file
View File

@ -0,0 +1,2 @@
FROM rabbitmq:3.7.9-management
RUN rabbitmq-plugins enable rabbitmq_consistent_hash_exchange

17
dockerfiles/sprint/Dockerfile Normal file
View File

@ -0,0 +1,17 @@
FROM docker:dind
RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN apk update && apk add postgresql-dev gcc python3-dev musl-dev jpeg-dev zlib-dev libjpeg
RUN pip3 install --no-cache --upgrade pip setuptools
ENV PYTHONUNBUFFERED 1
ENV DJANGO_SETTINGS_MODULE Sprint.settings
RUN mkdir -p /usr/src/app/
WORKDIR /usr/src/app/
COPY ../.. /usr/src/app/
RUN pip3 install -r requirements.txt
EXPOSE 8000

3
dockerfiles/worker/Dockerfile Normal file
View File

@ -0,0 +1,3 @@
FROM docker:dind
WORKDIR /usr/src/app/

98
pg_hba.conf Normal file
View File

@ -0,0 +1,98 @@
# PostgreSQL Client Authentication Configuration File
# ===================================================
#
# Refer to the "Client Authentication" section in the PostgreSQL
# documentation for a complete description of this file. A short
# synopsis follows.
#
# This file controls: which hosts are allowed to connect, how clients
# are authenticated, which PostgreSQL user names they can use, which
# databases they can access. Records take one of these forms:
#
# local DATABASE USER METHOD [OPTIONS]
# host DATABASE USER ADDRESS METHOD [OPTIONS]
# hostssl DATABASE USER ADDRESS METHOD [OPTIONS]
# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS]
# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS]
# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS]
#
# (The uppercase items must be replaced by actual values.)
#
# The first field is the connection type: "local" is a Unix-domain
# socket, "host" is either a plain or SSL-encrypted TCP/IP socket,
# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a
# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a
# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a
# non-GSSAPI socket.
#
# DATABASE can be "all", "sameuser", "samerole", "replication", a
# database name, or a comma-separated list thereof. The "all"
# keyword does not match "replication". Access to replication
# must be enabled in a separate record (see example below).
#
# USER can be "all", a user name, a group name prefixed with "+", or a
# comma-separated list thereof. In both the DATABASE and USER fields
# you can also write a file name prefixed with "@" to include names
# from a separate file.
#
# ADDRESS specifies the set of hosts the record matches. It can be a
# host name, or it is made up of an IP address and a CIDR mask that is
# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
# specifies the number of significant bits in the mask. A host name
# that starts with a dot (.) matches a suffix of the actual host name.
# Alternatively, you can write an IP address and netmask in separate
# columns to specify the set of hosts. Instead of a CIDR-address, you
# can write "samehost" to match any of the server's own IP addresses,
# or "samenet" to match any address in any subnet that the server is
# directly connected to.
#
# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
# Note that "password" sends passwords in clear text; "md5" or
# "scram-sha-256" are preferred since they send encrypted passwords.
#
# OPTIONS are a set of options for the authentication in the format
# NAME=VALUE. The available options depend on the different
# authentication methods -- refer to the "Client Authentication"
# section in the documentation for a list of which options are
# available for which authentication methods.
#
# Database and user names containing spaces, commas, quotes and other
# special characters must be quoted. Quoting one of the keywords
# "all", "sameuser", "samerole" or "replication" makes the name lose
# its special character, and just match a database or username with
# that name.
#
# This file is read on server startup and when the server receives a
# SIGHUP signal. If you edit the file on a running system, you have to
# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
# or execute "SELECT pg_reload_conf()".
#
# Put your actual configuration here
# ----------------------------------
#
# If you want to allow non-local connections, you need to add more
# "host" records. In that case you will also need to make PostgreSQL
# listen on a non-local interface via the listen_addresses
# configuration parameter, or via the -i or -h command line switches.
# CAUTION: Configuring the system for local "trust" authentication
# allows any local user to connect as any PostgreSQL user, including
# the database superuser. If you do not trust all your local users,
# use another authentication method.
# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
host all all 127.0.0.1/32 trust
# IPv6 local connections:
host all all ::1/128 trust
# Allow replication connections from localhost, by a user with the
# replication privilege.
local replication all trust
host replication all 127.0.0.1/32 trust
host replication all ::1/128 trust
host all all 0.0.0.0/0 trust

13
requirements.txt
View File

@ -1,9 +1,9 @@
amqp==5.0.6
anyjson==0.3.3
asgiref==3.3.4
billiard==3.6.4.0
cached-property==1.5.2
celery==5.2.0b2
certifi==2021.5.30
charset-normalizer==2.0.4
click==8.0.1
click-didyoumean==0.0.3
click-plugins==1.1.1
@ -11,18 +11,23 @@ click-repl==0.2.0
dj-database-url==0.5.0
Django==3.2.4
gunicorn==20.1.0
idna==3.2
importlib-metadata==4.5.0
kombu==5.1.0
numpy==1.20.3
pandas==1.2.4
pika==1.2.0
Pillow==8.3.1
prompt-toolkit==3.0.18
psycopg2==2.9.1
psycopg2-binary==2.9.1
pyTelegramBotAPI==4.1.1
python-dateutil==2.8.1
pytz==2021.1
redis==3.5.3
requests==2.26.0
six==1.16.0
sqlparse==0.4.1
typing-extensions==3.10.0.0
urllib3==1.26.6
vine==5.0.0
wcwidth==0.2.5
zipp==3.5.0

6
scripts/create_worker.sh Executable file
View File

@ -0,0 +1,6 @@
apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
python3 -m ensurepip
apk update && apk add postgresql-dev gcc python3-dev musl-dev jpeg-dev zlib-dev libjpeg
pip3 install --no-cache --upgrade pip setuptools
pip3 install -r requirements.txt
python3 manage.py receive

3
scripts/runserver.sh Executable file
View File

@ -0,0 +1,3 @@
python manage.py migrate
python manage.py update_languages
python manage.py runserver 0.0.0.0:8000

5
setup.sh
View File

@ -1,5 +0,0 @@
pip3 install --upgrade pip wheel setuptools
pip3 install venv
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt

36
start.py
View File

@ -1,36 +0,0 @@
#! /usr/bin/env python
import asyncio
import sys
from os import listdir
async def execute(service):
while True:
try:
mod = __import__(f'daemons.{service}')
module = getattr(mod, service)
daemon = getattr(module, 'Daemon')
await daemon().execute()
except Exception as e:
print(e)
finally:
await asyncio.sleep(5)
async def main():
if sys.argv[1] == "--all":
services = list(
map(
lambda x: x[:-3],
[file for file in listdir("daemons") if file.endswith(".py") and file != '__init__.py'],
)
)
else:
services = sys.argv[1:]
await asyncio.gather(*[execute(service) for service in services])
loop = asyncio.get_event_loop()
loop.run_until_complete(main())

8
venv/bin/f2py
View File

@ -1,8 +0,0 @@
#!/Users/egormatveev/Sprint/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from numpy.f2py.f2py2e import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

8
venv/bin/f2py3
View File

@ -1,8 +0,0 @@
#!/Users/egormatveev/Sprint/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from numpy.f2py.f2py2e import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

8
venv/bin/f2py3.9
View File

@ -1,8 +0,0 @@
#!/Users/egormatveev/Sprint/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from numpy.f2py.f2py2e import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())