From 85e7957d60c001f2b22ad512ee1045e054823805 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Sun, 1 May 2022 19:30:38 +0300 Subject: [PATCH] tempfile --- .gitlab-ci.yml | 5 -- Sprint/settings.py | 2 - SprintLib/testers/BaseTester.py | 140 ++++++++++++++++---------------- docker-compose-deploy.yaml | 12 +-- 4 files changed, 69 insertions(+), 90 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6962fee..9a6179e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,6 @@ stages: - deploy-dev - deploy-prod -variables: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" - build: stage: build tags: @@ -34,7 +31,6 @@ deploy-dev: - when: manual variables: PORT: 80 - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_PASSWORD: "$DB_PASSWORD_DEMO" DEBUG: "true" TELEGRAM_TOKEN: "$TELEGRAM_TOKEN_DEMO" @@ -50,6 +46,5 @@ deploy-prod: when: manual variables: PORT: 80 - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_PASSWORD: "$DB_PASSWORD_PROD" TELEGRAM_TOKEN: "$TELEGRAM_TOKEN_PROD" diff --git a/Sprint/settings.py b/Sprint/settings.py index 4ae38a2..be2671c 100644 --- a/Sprint/settings.py +++ b/Sprint/settings.py @@ -136,12 +136,10 @@ MEDIA_URL = "/media/" STATIC_ROOT = os.path.join(BASE_DIR, "static") DATA_ROOT = os.path.join(BASE_DIR, "data") EXTRA_FILES_ROOT = os.path.join(BASE_DIR, "extra_files") -SOLUTIONS_ROOT_EXTERNAL = os.getenv("SOLUTIONS_ROOT_EXTERNAL", os.path.join(DATA_ROOT, "solutions")) for root in DATA_ROOT, EXTRA_FILES_ROOT: if not os.path.exists(root): os.mkdir(root) -SOLUTIONS_ROOT = os.path.join(DATA_ROOT, "solutions") RABBIT_HOST = os.getenv("RABBIT_HOST", "127.0.0.1") RABBIT_PORT = 5672 diff --git a/SprintLib/testers/BaseTester.py b/SprintLib/testers/BaseTester.py index 07f19ba..50813eb 100644 --- a/SprintLib/testers/BaseTester.py +++ b/SprintLib/testers/BaseTester.py @@ -1,6 +1,7 @@ from os import listdir, mkdir from os.path import join, exists from subprocess import call, TimeoutExpired +from tempfile import TemporaryDirectory from SprintLib.queue import notify, send_to_queue from Main.models import ExtraFile, SolutionFile @@ -16,6 +17,7 @@ class TestException(Exception): class BaseTester: working_directory = "app" checker_code = None + path = "" def before_test(self): files = [ @@ -70,10 +72,6 @@ class BaseTester: def call(self, command): return call(f'cd {self.path} && {command}', shell=True) - @property - def path(self): - return join("solutions", str(self.solution.id)) - def __init__(self, solution): self.solution = solution @@ -102,75 +100,73 @@ class BaseTester: def execute(self): self.solution.result = CONSTS["testing_status"] self.solution.save() - if not exists("solutions"): - mkdir("solutions") - mkdir(self.path) - for file in SolutionFile.objects.filter(solution=self.solution): - dirs = file.path.split("/") - for i in range(len(dirs) - 1): - name = join( - self.path, "/".join(dirs[: i + 1]) + with TemporaryDirectory() as self.path: + for file in SolutionFile.objects.filter(solution=self.solution): + dirs = file.path.split("/") + for i in range(len(dirs) - 1): + name = join( + self.path, "/".join(dirs[: i + 1]) + ) + if not exists(name): + mkdir(name) + with open( + join(self.path, file.path), "wb" + ) as fs: + fs.write(get_bytes(file.fs_id).replace(b"\r\n", b"\n")) + for file in ExtraFile.objects.filter(task=self.solution.task): + with open( + join(self.path, file.filename), 'wb' + ) as fs: + bts = get_bytes(file.fs_id) + fs.write(bts) + print("Files copied") + self._setup_networking() + docker_command = f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id} --volume=/sprint-data/solutions/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}" + print(docker_command) + call(docker_command, shell=True) + checker = ExtraFile.objects.filter(task=self.solution.task, filename='checker.py').first() + if checker is not None: + self.checker_code = checker.text + call(f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id}_checker --volume=/sprint-data/solutions/{self.solution.id}:/app -t -d python:3.6", shell=True) + print("Container created") + try: + self.before_test() + print("before test finished") + for test in self.solution.task.tests: + if not test.filename.endswith(".a"): + self.predicted = ExtraFile.objects.get( + task=self.solution.task, filename=test.filename + ".a" + ).text.strip().replace('\r\n', '\n') + print('predicted:', self.predicted) + self.solution.test = int(test.filename) + self.solution.extras[test.filename] = {'predicted': self.predicted, 'output': ''} + self.solution.save() + try: + self.test(test.filename) + finally: + if exists(join(self.path, "output.txt")): + try: + self.solution.extras[test.filename]['output'] = open(join(self.path, 'output.txt'), 'r').read() + except UnicodeDecodeError: + self.solution.extras[test.filename]['output'] = '' + self.after_test() + self.solution.result = CONSTS["ok_status"] + self.solution.test = None + progress = Progress.objects.get( + user=self.solution.user, task=self.solution.task ) - if not exists(name): - mkdir(name) - with open( - join(self.path, file.path), "wb" - ) as fs: - fs.write(get_bytes(file.fs_id).replace(b"\r\n", b"\n")) - for file in ExtraFile.objects.filter(task=self.solution.task): - with open( - join(self.path, file.filename), 'wb' - ) as fs: - bts = get_bytes(file.fs_id) - fs.write(bts) - print("Files copied") - self._setup_networking() - docker_command = f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id} --volume=/sprint-data/solutions/{self.solution.id}:/{self.working_directory} -t -d {self.solution.language.image}" - print(docker_command) - call(docker_command, shell=True) - checker = ExtraFile.objects.filter(task=self.solution.task, filename='checker.py').first() - if checker is not None: - self.checker_code = checker.text - call(f"docker run --network solution_network_{self.solution.id} --name solution_{self.solution.id}_checker --volume=/sprint-data/solutions/{self.solution.id}:/app -t -d python:3.6", shell=True) - print("Container created") - try: - self.before_test() - print("before test finished") - for test in self.solution.task.tests: - if not test.filename.endswith(".a"): - self.predicted = ExtraFile.objects.get( - task=self.solution.task, filename=test.filename + ".a" - ).text.strip().replace('\r\n', '\n') - print('predicted:', self.predicted) - self.solution.test = int(test.filename) - self.solution.extras[test.filename] = {'predicted': self.predicted, 'output': ''} - self.solution.save() - try: - self.test(test.filename) - finally: - if exists(join(self.path, "output.txt")): - try: - self.solution.extras[test.filename]['output'] = open(join(self.path, 'output.txt'), 'r').read() - except UnicodeDecodeError: - self.solution.extras[test.filename]['output'] = '' - self.after_test() - self.solution.result = CONSTS["ok_status"] - self.solution.test = None - progress = Progress.objects.get( - user=self.solution.user, task=self.solution.task - ) - if progress.finished_time is None: - progress.finished_time = self.solution.time_sent - progress.finished = True - progress.save() - progress.increment_rating() - except TestException as e: - self.solution.result = str(e) - except TimeoutExpired: - self.solution.result = "TL" - except Exception as e: - self.solution.result = "TE" - raise e + if progress.finished_time is None: + progress.finished_time = self.solution.time_sent + progress.finished = True + progress.save() + progress.increment_rating() + except TestException as e: + self.solution.result = str(e) + except TimeoutExpired: + self.solution.result = "TL" + except Exception as e: + self.solution.result = "TE" + print(e) self.solution.save() send_to_queue("cleaner", {"type": "container", "name": f"solution_{self.solution.id}"}) send_to_queue("cleaner", {"type": "container", "name": f"solution_{self.solution.id}_checker"}) diff --git a/docker-compose-deploy.yaml b/docker-compose-deploy.yaml index 85df67d..4e2957b 100644 --- a/docker-compose-deploy.yaml +++ b/docker-compose-deploy.yaml @@ -57,7 +57,6 @@ services: networks: - net environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -74,7 +73,6 @@ services: networks: - net environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -94,7 +92,6 @@ services: networks: - net environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -159,7 +156,6 @@ services: - net command: ./manage.py storage environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -181,7 +177,6 @@ services: networks: - net environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -202,7 +197,6 @@ services: networks: - net environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -224,7 +218,6 @@ services: - net command: ./manage.py receive environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -232,7 +225,7 @@ services: DEBUG: $DEBUG TELEGRAM_TOKEN: $TELEGRAM_TOKEN volumes: - - /sprint-data/solutions:/usr/src/app/solutions + - /var/folders:/var/folders - /var/run/docker.sock:/var/run/docker.sock deploy: mode: replicated @@ -272,7 +265,6 @@ services: - net command: ./manage.py file_generator environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -294,7 +286,6 @@ services: - net command: ./manage.py notification_manager environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq" @@ -316,7 +307,6 @@ services: - net command: ./manage.py apply_languages environment: - SOLUTIONS_ROOT_EXTERNAL: "/sprint-data/data/solutions" DB_HOST: "postgres" FS_HOST: "storage" RABBIT_HOST: "rabbitmq"