From 928cc4489d10b0de542180843223d0c929069ef3 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 2 Nov 2021 23:33:38 +0300 Subject: [PATCH] tester in docker works --- Dockerfile | 13 ++- {daemons => Main/management}/__init__.py | 0 Main/management/commands/__init__.py | 0 Main/management/commands/receive.py | 32 +++++++ Main/management/commands/test.py | 15 +++ Main/management/commands/update_languages.py | 8 ++ Main/models/solution.py | 8 +- Main/tasks.py | 9 -- Main/views/TaskView.py | 10 +- Sprint/__init__.py | 3 - Sprint/celery.py | 17 ---- Sprint/settings.py | 18 ++-- SprintLib/BaseDaemon.py | 17 ---- SprintLib/queue.py | 12 +++ SprintLib/testers/BaseTester.py | 55 +++++++---- SprintLib/testers/Python3Tester.py | 2 +- apply_models.sh | 3 - daemons/bot.py | 6 -- daemons/celery.py | 6 -- daemons/redis.py | 6 -- daemons/web.py | 6 -- docker-compose.yaml | 50 +++++++++- dockerfiles/postgres/Dockerfile | 2 + dockerfiles/rabbitmq/Dockerfile | 2 + dockerfiles/sprint/Dockerfile | 17 ++++ dockerfiles/worker/Dockerfile | 3 + pg_hba.conf | 98 ++++++++++++++++++++ requirements.txt | 13 ++- scripts/create_worker.sh | 6 ++ scripts/runserver.sh | 3 + setup.sh | 5 - start.py | 36 ------- venv/bin/f2py | 8 -- venv/bin/f2py3 | 8 -- venv/bin/f2py3.9 | 8 -- 35 files changed, 321 insertions(+), 184 deletions(-) rename {daemons => Main/management}/__init__.py (100%) create mode 100644 Main/management/commands/__init__.py create mode 100644 Main/management/commands/receive.py create mode 100644 Main/management/commands/test.py create mode 100644 Main/management/commands/update_languages.py delete mode 100644 Main/tasks.py delete mode 100644 Sprint/celery.py delete mode 100644 SprintLib/BaseDaemon.py create mode 100644 SprintLib/queue.py delete mode 100644 apply_models.sh delete mode 100644 daemons/bot.py delete mode 100644 daemons/celery.py delete mode 100644 daemons/redis.py delete mode 100644 daemons/web.py create mode 100644 dockerfiles/postgres/Dockerfile create mode 100644 dockerfiles/rabbitmq/Dockerfile create mode 100644 dockerfiles/sprint/Dockerfile create mode 100644 dockerfiles/worker/Dockerfile create mode 100644 pg_hba.conf create mode 100755 scripts/create_worker.sh create mode 100755 scripts/runserver.sh delete mode 100644 setup.sh delete mode 100755 start.py delete mode 100755 venv/bin/f2py delete mode 100755 venv/bin/f2py3 delete mode 100755 venv/bin/f2py3.9 diff --git a/Dockerfile b/Dockerfile index 0c901ef..496f4ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,18 @@ -FROM python:3.7 +FROM docker:dind + +RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN apk update && apk add postgresql-dev gcc python3-dev musl-dev jpeg-dev zlib-dev libjpeg +RUN pip3 install --no-cache --upgrade pip setuptools +RUN addgroup -S docker ENV PYTHONUNBUFFERED 1 ENV DJANGO_SETTINGS_MODULE Sprint.settings RUN mkdir -p /usr/src/app/ WORKDIR /usr/src/app/ -COPY . /usr/src/app/ +COPY ../.. /usr/src/app/ -RUN pip install --upgrade pip wheel setuptools -RUN pip install -r requirements.txt +RUN pip3 install -r requirements.txt EXPOSE 8000 diff --git a/daemons/__init__.py b/Main/management/__init__.py similarity index 100% rename from daemons/__init__.py rename to Main/management/__init__.py diff --git a/Main/management/commands/__init__.py b/Main/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Main/management/commands/receive.py b/Main/management/commands/receive.py new file mode 100644 index 0000000..46f278e --- /dev/null +++ b/Main/management/commands/receive.py @@ -0,0 +1,32 @@ +import pika +from django.core.management.base import BaseCommand + +from Main.models import Solution +from Sprint import settings +from SprintLib.testers import * + + +class Command(BaseCommand): + help = 'Tests solution' + + def handle(self, *args, **options): + print("Enter worker") + connection = pika.BlockingConnection(pika.ConnectionParameters(host=settings.RABBIT_HOST)) + channel = connection.channel() + channel.queue_declare(queue='test') + + def callback(ch, method, properties, body): + try: + id = int(str(body, encoding='utf-8')) + print(f"Received id {id}") + solution = Solution.objects.get(id=id) + except: + return + try: + eval(solution.language.work_name + 'Tester')(solution).execute() + except: + solution.result = 'TE' + solution.save() + + channel.basic_consume(queue='test', on_message_callback=callback, auto_ack=True) + channel.start_consuming() diff --git a/Main/management/commands/test.py b/Main/management/commands/test.py new file mode 100644 index 0000000..6eff05f --- /dev/null +++ b/Main/management/commands/test.py @@ -0,0 +1,15 @@ +from django.core.management.base import BaseCommand + +from Main.models import Solution +from SprintLib.testers import * + + +class Command(BaseCommand): + help = 'Tests solution' + + def add_arguments(self, parser): + parser.add_argument('solution_id', type=int) + + def handle(self, *args, **options): + solution = Solution.objects.get(id=options['solution_id']) + eval(solution.language.work_name + 'Tester')(solution).execute() diff --git a/Main/management/commands/update_languages.py b/Main/management/commands/update_languages.py new file mode 100644 index 0000000..4041058 --- /dev/null +++ b/Main/management/commands/update_languages.py @@ -0,0 +1,8 @@ +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = 'Updates languages' + + def handle(self, *args, **options): + pass diff --git a/Main/models/solution.py b/Main/models/solution.py index 6b205bd..518ab68 100644 --- a/Main/models/solution.py +++ b/Main/models/solution.py @@ -10,7 +10,7 @@ from django.utils import timezone from Main.models.task import Task from Main.models.language import Language -from Sprint.settings import CONSTS, SOLUTIONS_ROOT +from Sprint.settings import CONSTS, SOLUTIONS_ROOT, SOLUTIONS_ROOT_EXTERNAL class Solution(models.Model): @@ -61,5 +61,9 @@ class Solution(models.Model): def testing_directory(self): return join(self.directory, 'test_dir') + @property + def volume_directory(self): + return join(SOLUTIONS_ROOT_EXTERNAL, str(self.id), 'test_dir') + def exec_command(self, command, working_directory='app', timeout=None): - return call(f'docker exec -i solution_{self.id} bash -c "cd {working_directory} && {command}"', shell=True, timeout=timeout) + return call(f'docker exec -i solution_{self.id} sh -c "cd {working_directory} && {command}"', shell=True, timeout=timeout) diff --git a/Main/tasks.py b/Main/tasks.py deleted file mode 100644 index c61c472..0000000 --- a/Main/tasks.py +++ /dev/null @@ -1,9 +0,0 @@ -from Main.models import Solution -from Sprint.celery import app -from SprintLib.testers import * - - -@app.task -def start_testing(solution_id): - solution = Solution.objects.get(id=solution_id) - eval(solution.language.work_name + 'Tester')(solution).execute() diff --git a/Main/views/TaskView.py b/Main/views/TaskView.py index 38ee2a2..9c4d9b2 100644 --- a/Main/views/TaskView.py +++ b/Main/views/TaskView.py @@ -1,8 +1,8 @@ from zipfile import ZipFile -from Main.models import Solution, Progress -from Main.tasks import start_testing +from Main.models import Solution from SprintLib.BaseView import BaseView, Language +from SprintLib.queue import send_testing from SprintLib.testers import * @@ -31,11 +31,13 @@ class TaskView(BaseView): file_path = join(self.solution.directory, filename) with open(file_path, 'w') as fs: fs.write(self.request.POST['code']) - start_testing.delay(self.solution.id) + send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) def post_1(self): # отправка решения через файл + if 'file' not in self.request.FILES: + return "task?task_id=" + str(self.entities.task.id) filename = self.request.FILES['file'].name file_path = join(self.solution.directory, filename) with open(file_path, 'wb') as fs: @@ -44,5 +46,5 @@ class TaskView(BaseView): if filename.endswith('.zip'): with ZipFile(file_path) as obj: obj.extractall(self.solution.directory) - start_testing.delay(self.solution.id) + send_testing(self.solution.id) return "task?task_id=" + str(self.entities.task.id) diff --git a/Sprint/__init__.py b/Sprint/__init__.py index 711580c..e69de29 100644 --- a/Sprint/__init__.py +++ b/Sprint/__init__.py @@ -1,3 +0,0 @@ -from .celery import app as celery - -__all__ = ('celery',) diff --git a/Sprint/celery.py b/Sprint/celery.py deleted file mode 100644 index da680ac..0000000 --- a/Sprint/celery.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -from celery import Celery - -# Set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Sprint.settings') - -app = Celery('Sprint') - -# Using a string here means the worker doesn't have to serialize -# the configuration object to child processes. -# - namespace='CELERY' means all celery-related configuration keys -# should have a `CELERY_` prefix. -app.config_from_object('django.conf:settings', namespace='CELERY') - -# Load task modules from all registered Django apps. -app.autodiscover_tasks() diff --git a/Sprint/settings.py b/Sprint/settings.py index a2d2129..60efeb2 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -84,11 +84,11 @@ WSGI_APPLICATION = "Sprint.wsgi.application" DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": "postgres", + "NAME": "sprint", "USER": "postgres", "PASSWORD": "password", - "HOST": "0.0.0.0", - "PORT": "5432", + "HOST": "postgres", + "PORT": 5432, } } @@ -141,6 +141,12 @@ MEDIA_ROOT = os.path.join(BASE_DIR, "media") DATA_ROOT = os.path.join(BASE_DIR, "data") SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") +SOLUTIONS_ROOT_EXTERNAL = "/Users/egormatveev/Sprint/data/solutions" + +RABBIT_HOST = "rabbitmq" +RABBIT_PORT = 5672 + + STATICFILES_DIRS = [ os.path.join(BASE_DIR, "Main/static"), ] @@ -149,12 +155,6 @@ STATICFILES_DIRS = [ # Authentication backends AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",) - -# Celery Configuration Options -CELERY_TIMEZONE = "Europe/Moscow" -CELERY_TASK_TRACK_STARTED = True -CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' - CONSTS = { "online_status": "Online", "in_queue_status": "In queue", diff --git a/SprintLib/BaseDaemon.py b/SprintLib/BaseDaemon.py deleted file mode 100644 index ba288e4..0000000 --- a/SprintLib/BaseDaemon.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio -import sys - - -class BaseDaemon: - def command(self): - raise NotImplementedError() - - async def execute(self): - cmd = self.command() - proc = await asyncio.create_subprocess_shell(cmd, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr) - stdout, stderr = await proc.communicate() - print(f"[{cmd!r} exited with {proc.returncode}]") - if stdout: - print(f"[stdout]\n{stdout.decode()}") - if stderr: - print(f"[stderr]\n{stderr.decode()}") diff --git a/SprintLib/queue.py b/SprintLib/queue.py new file mode 100644 index 0000000..8b4bbc5 --- /dev/null +++ b/SprintLib/queue.py @@ -0,0 +1,12 @@ +import pika + +from Sprint import settings + + +def send_testing(solution_id): + with pika.BlockingConnection( + pika.ConnectionParameters(host=settings.RABBIT_HOST, port=settings.RABBIT_PORT) + ) as connection: + channel = connection.channel() + channel.queue_declare(queue="test") + channel.basic_publish(exchange="", routing_key="test", body=bytes(str(solution_id), encoding='utf-8')) diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index c66d8f7..10347d8 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -1,5 +1,5 @@ -from os import listdir -from os.path import join +from os import listdir, mkdir +from os.path import join, exists from shutil import copyfile, rmtree from subprocess import call, TimeoutExpired @@ -18,10 +18,17 @@ class BaseTester: working_directory = "app" def before_test(self): - files = [file for file in listdir(self.solution.testing_directory) if file.endswith('.' + self.solution.language.file_type)] - code = self.solution.exec_command(f'{self.build_command} {" ".join(files)}', working_directory=self.working_directory) + files = [ + file + for file in listdir(self.solution.testing_directory) + if file.endswith("." + self.solution.language.file_type) + ] + code = self.solution.exec_command( + f'{self.build_command} {" ".join(files)}', + working_directory=self.working_directory, + ) if code != 0: - raise TestException('CE') + raise TestException("CE") def test(self, filename): code = self.solution.exec_command( @@ -30,9 +37,7 @@ class BaseTester: ) if code != 0: raise TestException("RE") - result = open( - join(self.solution.testing_directory, "output.txt"), "r" - ).read() + result = open(join(self.solution.testing_directory, "output.txt"), "r").read() if result.strip() != self.predicted.strip(): raise TestException("WA") @@ -51,17 +56,23 @@ class BaseTester: self.solution = solution def execute(self): - copy_content(self.solution.directory, self.solution.testing_directory, ('test_dir',)) + if not exists(self.solution.testing_directory): + mkdir(self.solution.testing_directory) + copy_content( + self.solution.directory, self.solution.testing_directory, ("test_dir",) + ) self.solution.result = CONSTS["testing_status"] self.solution.save() - call( - f"docker run --name solution_{self.solution.id} --volume={self.solution.testing_directory}:/{self.working_directory} -t -d {self.solution.language.image}", - shell=True, - ) + docker_command = f"docker run --name solution_{self.solution.id} --volume={self.solution.volume_directory}:/{self.working_directory} -t -d {self.solution.language.image}" + print(docker_command) + call(docker_command, shell=True) + print("Container created") for file in ExtraFile.objects.filter(task=self.solution.task): copyfile(file.path, join(self.solution.testing_directory, file.filename)) + print("Files copied") try: self.before_test() + print("before test finished") for test in self.solution.task.tests: if not test.filename.endswith(".a"): self.predicted = ExtraFile.objects.get( @@ -70,7 +81,9 @@ class BaseTester: self.test(test.filename) self.after_test() self.solution.result = CONSTS["ok_status"] - progress = Progress.objects.get(user=self.solution.user, task=self.solution.task) + progress = Progress.objects.get( + user=self.solution.user, task=self.solution.task + ) if progress.finished_time is None: progress.finished_time = self.solution.time_sent progress.finished = True @@ -88,9 +101,11 @@ class BaseTester: rmtree(self.solution.testing_directory) self.solution.user.userinfo.refresh_from_db() if self.solution.user.userinfo.notification_solution_result: - bot.send_message(self.solution.user.userinfo.telegram_chat_id, - f'Задача: {self.solution.task.name}\n' - f'Результат: {self.solution.result}\n' - f'Очки решения: {Progress.by_solution(self.solution).score}\n' - f'Текущий рейтинг: {self.solution.user.userinfo.rating}', - parse_mode='html') + bot.send_message( + self.solution.user.userinfo.telegram_chat_id, + f"Задача: {self.solution.task.name}\n" + f"Результат: {self.solution.result}\n" + f"Очки решения: {Progress.by_solution(self.solution).score}\n" + f"Текущий рейтинг: {self.solution.user.userinfo.rating}", + parse_mode="html", + ) diff --git a/SprintLib/testers/Python3Tester.py b/SprintLib/testers/Python3Tester.py index 691181e..1ca8ada 100644 --- a/SprintLib/testers/Python3Tester.py +++ b/SprintLib/testers/Python3Tester.py @@ -16,4 +16,4 @@ class Python3Tester(BaseTester): @property def command(self): - return f"python {self.file}" + return f"python3 {self.file}" diff --git a/apply_models.sh b/apply_models.sh deleted file mode 100644 index e78480e..0000000 --- a/apply_models.sh +++ /dev/null @@ -1,3 +0,0 @@ -source venv/bin/activate -python manage.py makemigrations -python manage.py migrate \ No newline at end of file diff --git a/daemons/bot.py b/daemons/bot.py deleted file mode 100644 index f73667b..0000000 --- a/daemons/bot.py +++ /dev/null @@ -1,6 +0,0 @@ -from SprintLib.BaseDaemon import BaseDaemon - - -class Daemon(BaseDaemon): - def command(self): - return "python bot.py" diff --git a/daemons/celery.py b/daemons/celery.py deleted file mode 100644 index 6cdc7f5..0000000 --- a/daemons/celery.py +++ /dev/null @@ -1,6 +0,0 @@ -from SprintLib.BaseDaemon import BaseDaemon - - -class Daemon(BaseDaemon): - def command(self): - return "celery -A Sprint worker -l INFO --concurrency=4" diff --git a/daemons/redis.py b/daemons/redis.py deleted file mode 100644 index 59045b8..0000000 --- a/daemons/redis.py +++ /dev/null @@ -1,6 +0,0 @@ -from SprintLib.BaseDaemon import BaseDaemon - - -class Daemon(BaseDaemon): - def command(self): - return "redis-server" diff --git a/daemons/web.py b/daemons/web.py deleted file mode 100644 index dbeacb0..0000000 --- a/daemons/web.py +++ /dev/null @@ -1,6 +0,0 @@ -from SprintLib.BaseDaemon import BaseDaemon - - -class Daemon(BaseDaemon): - def command(self): - return "python manage.py runserver 0.0.0.0:80" diff --git a/docker-compose.yaml b/docker-compose.yaml index 5e5e194..537373a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,11 +5,57 @@ services: postgres: restart: always - image: postgres - + build: + context: . + dockerfile: dockerfiles/postgres/Dockerfile environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: password POSTGRES_DB: postgres + volumes: + - ./postgres-data:/var/lib/postgresql/data ports: - "5432:5432" + + web: + image: sprint + restart: always + command: scripts/runserver.sh + ports: + - "8000:8000" + volumes: + - .:/usr/src/app + depends_on: + - postgres + - rabbitmq + + bot: + image: sprint + restart: always + command: python bot.py + depends_on: + - web + + rabbitmq: + restart: always + build: + context: . + dockerfile: dockerfiles/rabbitmq/Dockerfile + ports: + - "15672:15672" + - "5672:5672" + + worker: + restart: always + build: + context: . + dockerfile: dockerfiles/worker/Dockerfile + privileged: true + command: scripts/create_worker.sh + depends_on: + - web + - rabbitmq + - postgres + volumes: + - .:/usr/src/app + - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/dockerfiles/postgres/Dockerfile b/dockerfiles/postgres/Dockerfile new file mode 100644 index 0000000..e302d8f --- /dev/null +++ b/dockerfiles/postgres/Dockerfile @@ -0,0 +1,2 @@ +FROM postgres +COPY ../../pg_hba.conf /var/lib/postgresql/data/pg_hba.conf diff --git a/dockerfiles/rabbitmq/Dockerfile b/dockerfiles/rabbitmq/Dockerfile new file mode 100644 index 0000000..4e2f27d --- /dev/null +++ b/dockerfiles/rabbitmq/Dockerfile @@ -0,0 +1,2 @@ +FROM rabbitmq:3.7.9-management +RUN rabbitmq-plugins enable rabbitmq_consistent_hash_exchange diff --git a/dockerfiles/sprint/Dockerfile b/dockerfiles/sprint/Dockerfile new file mode 100644 index 0000000..129c50e --- /dev/null +++ b/dockerfiles/sprint/Dockerfile @@ -0,0 +1,17 @@ +FROM docker:dind + +RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN apk update && apk add postgresql-dev gcc python3-dev musl-dev jpeg-dev zlib-dev libjpeg +RUN pip3 install --no-cache --upgrade pip setuptools + +ENV PYTHONUNBUFFERED 1 +ENV DJANGO_SETTINGS_MODULE Sprint.settings +RUN mkdir -p /usr/src/app/ +WORKDIR /usr/src/app/ + +COPY ../.. /usr/src/app/ + +RUN pip3 install -r requirements.txt + +EXPOSE 8000 diff --git a/dockerfiles/worker/Dockerfile b/dockerfiles/worker/Dockerfile new file mode 100644 index 0000000..ff96e78 --- /dev/null +++ b/dockerfiles/worker/Dockerfile @@ -0,0 +1,3 @@ +FROM docker:dind + +WORKDIR /usr/src/app/ \ No newline at end of file diff --git a/pg_hba.conf b/pg_hba.conf new file mode 100644 index 0000000..2761624 --- /dev/null +++ b/pg_hba.conf @@ -0,0 +1,98 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a +# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a +# non-GSSAPI socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# CAUTION: Configuring the system for local "trust" authentication +# allows any local user to connect as any PostgreSQL user, including +# the database superuser. If you do not trust all your local users, +# use another authentication method. + + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust +# Allow replication connections from localhost, by a user with the +# replication privilege. +local replication all trust +host replication all 127.0.0.1/32 trust +host replication all ::1/128 trust +host all all 0.0.0.0/0 trust \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1eadca0..dbd0f50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ amqp==5.0.6 -anyjson==0.3.3 asgiref==3.3.4 billiard==3.6.4.0 cached-property==1.5.2 -celery==5.2.0b2 +certifi==2021.5.30 +charset-normalizer==2.0.4 click==8.0.1 click-didyoumean==0.0.3 click-plugins==1.1.1 @@ -11,18 +11,23 @@ click-repl==0.2.0 dj-database-url==0.5.0 Django==3.2.4 gunicorn==20.1.0 +idna==3.2 importlib-metadata==4.5.0 kombu==5.1.0 -numpy==1.20.3 -pandas==1.2.4 +pika==1.2.0 Pillow==8.3.1 prompt-toolkit==3.0.18 +psycopg2==2.9.1 +psycopg2-binary==2.9.1 +pyTelegramBotAPI==4.1.1 python-dateutil==2.8.1 pytz==2021.1 redis==3.5.3 +requests==2.26.0 six==1.16.0 sqlparse==0.4.1 typing-extensions==3.10.0.0 +urllib3==1.26.6 vine==5.0.0 wcwidth==0.2.5 zipp==3.5.0 diff --git a/scripts/create_worker.sh b/scripts/create_worker.sh new file mode 100755 index 0000000..9293e2f --- /dev/null +++ b/scripts/create_worker.sh @@ -0,0 +1,6 @@ +apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python +python3 -m ensurepip +apk update && apk add postgresql-dev gcc python3-dev musl-dev jpeg-dev zlib-dev libjpeg +pip3 install --no-cache --upgrade pip setuptools +pip3 install -r requirements.txt +python3 manage.py receive \ No newline at end of file diff --git a/scripts/runserver.sh b/scripts/runserver.sh new file mode 100755 index 0000000..e0d6436 --- /dev/null +++ b/scripts/runserver.sh @@ -0,0 +1,3 @@ +python manage.py migrate +python manage.py update_languages +python manage.py runserver 0.0.0.0:8000 \ No newline at end of file diff --git a/setup.sh b/setup.sh deleted file mode 100644 index 7ecffd4..0000000 --- a/setup.sh +++ /dev/null @@ -1,5 +0,0 @@ -pip3 install --upgrade pip wheel setuptools -pip3 install venv -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt diff --git a/start.py b/start.py deleted file mode 100755 index 9f940bc..0000000 --- a/start.py +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env python - - -import asyncio -import sys -from os import listdir - - -async def execute(service): - while True: - try: - mod = __import__(f'daemons.{service}') - module = getattr(mod, service) - daemon = getattr(module, 'Daemon') - await daemon().execute() - except Exception as e: - print(e) - finally: - await asyncio.sleep(5) - - -async def main(): - if sys.argv[1] == "--all": - services = list( - map( - lambda x: x[:-3], - [file for file in listdir("daemons") if file.endswith(".py") and file != '__init__.py'], - ) - ) - else: - services = sys.argv[1:] - await asyncio.gather(*[execute(service) for service in services]) - - -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) diff --git a/venv/bin/f2py b/venv/bin/f2py deleted file mode 100755 index 5d102b9..0000000 --- a/venv/bin/f2py +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/egormatveev/Sprint/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from numpy.f2py.f2py2e import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/f2py3 b/venv/bin/f2py3 deleted file mode 100755 index 5d102b9..0000000 --- a/venv/bin/f2py3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/egormatveev/Sprint/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from numpy.f2py.f2py2e import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/f2py3.9 b/venv/bin/f2py3.9 deleted file mode 100755 index 5d102b9..0000000 --- a/venv/bin/f2py3.9 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/egormatveev/Sprint/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from numpy.f2py.f2py2e import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main())