commit 6ff0f5fd4409e7c5388a605f94e8e8e982627c51 Author: Egor Matveev Date: Sat May 31 00:33:56 2025 +0300 initial diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..760344a Binary files /dev/null and b/.DS_Store differ diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml new file mode 100644 index 0000000..6549e29 --- /dev/null +++ b/.deploy/deploy-dev.yaml @@ -0,0 +1,24 @@ +version: "3.4" + + +services: + certupdater: + image: mathwave/sprint-repo:certupdater + command: worker + environment: + MONGO_HOST: "mongo.develop.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_DEV + STAGE: "development" + networks: + - configurator + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + +networks: + configurator: + external: true diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml new file mode 100644 index 0000000..d4fe2b4 --- /dev/null +++ b/.deploy/deploy-prod.yaml @@ -0,0 +1,45 @@ +version: "3.4" + + +services: + worker: + image: mathwave/sprint-repo:pizda-bot + command: worker + environment: + MONGO_HOST: "mongo.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_PROD + STAGE: "production" + networks: + - queues + - configurator + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + + pizda-bot-nginx: + image: mathwave/sprint-repo:pizda-bot + command: api + environment: + MONGO_HOST: "mongo.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_PROD + networks: + - common-infra-nginx + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + +networks: + common-infra-nginx: + external: true + queues: + external: true + configurator: + external: true diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml new file mode 100644 index 0000000..9515a0c --- /dev/null +++ b/.gitea/workflows/deploy-dev.yaml @@ -0,0 +1,43 @@ +name: Deploy Dev + +on: + pull_request: + branches: + - dev + types: [closed] + +jobs: + build: + name: Build + runs-on: [ dev ] + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: dev + - name: build + run: docker build -t mathwave/sprint-repo:pizda-bot . + push: + name: Push + runs-on: [ dev ] + needs: build + steps: + - name: push + run: docker push mathwave/sprint-repo:pizda-bot + deploy-dev: + name: Deploy dev + runs-on: [prod] + needs: push + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: dev + - name: deploy + env: + MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }} + run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml pizda-bot-development diff --git a/.gitea/workflows/deploy-prod.yaml b/.gitea/workflows/deploy-prod.yaml new file mode 100644 index 0000000..cd6e772 --- /dev/null +++ b/.gitea/workflows/deploy-prod.yaml @@ -0,0 +1,43 @@ +name: Deploy Prod + +on: + pull_request: + branches: + - prod + types: [closed] + +jobs: + build: + name: Build + runs-on: [ dev ] + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: prod + - name: build + run: docker build -t mathwave/sprint-repo:pizda-bot . + push: + name: Push + runs-on: [ dev ] + needs: build + steps: + - name: push + run: docker push mathwave/sprint-repo:pizda-bot + deploy-prod: + name: Deploy prod + runs-on: [prod] + needs: push + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: prod + - name: deploy + env: + MONGO_PASSWORD_PROD: ${{ secrets.MONGO_PASSWORD_PROD }} + run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-prod.yaml pizda-bot diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43f4b97 --- /dev/null +++ b/.gitignore @@ -0,0 +1,122 @@ +# Django # +*.log +*.pot +*.pyc +__pycache__ +db.sqlite3 +media +data +*/__pycache__ + +# Backup files # +*.bak + +# If you are using PyCharm # +.idea +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/gradle.xml +.idea/**/libraries +*.iws /out/ + +# Python # +*.py[cod] +*$py.class + +# Distribution / packaging +.Python build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +postgres-data + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +ENV/ +venv/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Sublime Text # +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# sftp configuration file +sftp-config.json + +# Package control specific files Package +Control.last-run +Control.ca-list +Control.ca-bundle +Control.system-ca-bundle +GitHub.sublime-settings + +# Visual Studio Code # +.vscode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history + +*pb2* +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..274578f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM docker:dind + +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache python3 py3-pip && ln -sf python3 /usr/bin/python +RUN python3 -m venv venv +RUN venv/bin/python3 -m ensurepip +RUN venv/bin/pip3 install --no-cache --upgrade pip setuptools + +ENTRYPOINT [ "venv/bin/python3", "main.py" ] \ No newline at end of file diff --git a/blob.py b/blob.py new file mode 100644 index 0000000..38fb9e6 --- /dev/null +++ b/blob.py @@ -0,0 +1,14 @@ +import os +from minio import Minio + +MINIO_HOST = os.getenv("MINIO_HOST", "localhost") + ":9000" +MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "serviceminioadmin") +MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin") + + +minio = Minio( + MINIO_HOST, + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=False +) diff --git a/configurator.py b/configurator.py new file mode 100644 index 0000000..28d6f7a --- /dev/null +++ b/configurator.py @@ -0,0 +1,88 @@ +import json +import os +import urllib.parse +from threading import Thread +from time import sleep + +from requests import get + + +class ConfiguratorClient: + def __init__(self, app_name: str, stage: str, need_poll: bool = True): + self.app_name = app_name + self.stage = stage + self.endpoint = 'http://configurator/' + self.fetch_url = urllib.parse.urljoin(self.endpoint, '/api/v1/fetch') + self.config_storage = {} + self.experiment_storage = {} + self.staff_storage = {} + self.poll_data() + if need_poll: + self.poll_data_in_thread() + + def poll_data_in_thread(self): + def inner(): + while True: + sleep(30) + self.fetch() + + Thread(target=inner, daemon=True).start() + + def poll_data(self): + self.fetch(with_exception=True) + + def request_with_retries(self, url, params, with_exception=False, retries_count=3): + exception_to_throw = None + for _ in range(retries_count): + try: + response = get( + url, + params=params + ) + if response.status_code == 200: + return response.json() + print(f'Failed to request {url}, status_code={response.status_code}') + exception_to_throw = Exception('Not 200 status') + except Exception as exc: + print(exc) + exception_to_throw = exc + sleep(1) + print(f'Failed fetching with retries: {url}, {params}') + if with_exception: + raise exception_to_throw + + def fetch(self, with_exception=False): + if self.stage == 'local': + local_platform = json.loads(open('local_platform.json', 'r').read()) + self.config_storage = local_platform['configs'] + self.experiment_storage = local_platform['experiments'] + self.staff_storage = { + key: set(value) + for key, value in local_platform['platform_staff'].items() + } + return + response_data = self.request_with_retries(self.fetch_url, { + 'project': self.app_name, + 'stage': self.stage, + }, with_exception) + self.config_storage = response_data['configs'] + self.experiment_storage = response_data['experiments'] + self.staff_storage = { + key: set(value) + for key, value in response_data['platform_staff'].items() + } + + def is_staff(self, **kwargs): + for key, value in kwargs.items(): + if value in self.staff_storage[key]: + return True + return False + + def get_config(self, name): + return self.config_storage[name] + + def get_experiment(self, name): + return self.experiment_storage[name] + + +configurator = ConfiguratorClient("certupdater", os.getenv("STAGE")) diff --git a/main.py b/main.py new file mode 100644 index 0000000..fac52d2 --- /dev/null +++ b/main.py @@ -0,0 +1,44 @@ +import datetime +import subprocess +import time +from configurator import configurator +from mongo import mongo +from blob import minio + + +class Response: + code: int + out: str + err: str + + +def call(command: str) -> Response: + p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) + resp = p.wait() + response = Response() + response.code = resp + response.out, response.err = p.stdout.read().decode('utf-8'), p.stderr.read().decode('utf-8') + return response + + +def get_hosts() -> list[str]: + return list(set(configurator.get_config("hosts") + ["platform.chocomarsh.com"])) + + +def update_host(host: str): + gen_cert = call(f"docker exec $(docker ps -q -f name=infra_nginx) certbot --nginx --email emmtvv@gmail.com --agree-tos -d \"{host}\"") + if gen_cert.code != 0: + print("failed generating certificate") + return + + + +while True: + now = datetime.datetime.now() + mongo_hosts = mongo.hosts + for host in get_hosts(): + if now() + datetime.timedelta(days=14) > mongo_hosts[host]["expire_time"]: + update_host(host) + print(f"Host {host} updated") + minio.put_object("certupdater", "nginx.conf", ) + time.sleep(5 * 60) \ No newline at end of file diff --git a/mongo.py b/mongo.py new file mode 100644 index 0000000..0fcb1ec --- /dev/null +++ b/mongo.py @@ -0,0 +1,41 @@ +import datetime +import pymongo +import os + +MONGO_USER = os.getenv("MONGO_USER", "mongo") +MONGO_PASSWORD = os.getenv("MONGO_PASSWORD", "password") +MONGO_HOST = os.getenv("MONGO_HOST", "localhost") + + +class Mongo: + def __init__(self): + url = f"mongodb://{MONGO_USER}:{MONGO_PASSWORD}@{MONGO_HOST}:27017/" + self.client: pymongo.MongoClient = pymongo.MongoClient(url) + self.database = self.client.get_database("certupdater") + self.hosts_collection.create_index([ + ("host", 1) + ]) + + def __getitem__(self, item): + return self.database.get_collection(item) + + @property + def hosts_collection(self): + return self["hosts"] + + @property + def hosts(self): + hosts = {} + for host in self.hosts_collection.find({}): + hosts[host["host"]] = host + return hosts + + def update_date(self, host: str): + now = datetime.datetime.now() + if self.hosts_collection.find_one({"host": host}): + self.hosts_collection.update_one({"host": host}, {"$set": {"update_time": now, "expire_time": now + datetime.timedelta(days=90)}}) + else: + self.hosts_collection.insert_one({"host": host, "update_time": now, "expire_time": now + datetime.timedelta(days=90)}) + + +mongo = Mongo() diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..2c5438d --- /dev/null +++ b/storage.py @@ -0,0 +1,42 @@ +from cachetools import TTLCache +import os + +from utils.mongo import mongo + +CACHE_SIZE = int(os.getenv("CACHE_SIZE", 1000)) +CACHE_TTL = int(os.getenv("CACHE_TTL", 3600)) + +cache = TTLCache(CACHE_SIZE, CACHE_TTL) + + +def get_chat_info(chat_id: int) -> dict: + cached_info = cache.get(chat_id) + if cached_info is not None: + return cached_info + mongo_info = mongo.chats_collection.find_one({"chat_id": chat_id}) + if mongo_info is not None: + cache[chat_id] = mongo_info + return mongo_info + chat_info = {"chat_id": chat_id, "state": "default", "probability": 100} + mongo.chats_collection.insert_one(chat_info) + cache[chat_id] = chat_info + return chat_info + + +def set_values(chat_id: int, **values): + cached_info = cache.get(chat_id) + if cached_info is None: + mongo_info = mongo.chats_collection.find_one({"chat_id": chat_id}) + if mongo_info is None: + chat_info = {"chat_id": chat_id, "state": "default", "probability": 100} + chat_info.update(values) + mongo.chats_collection.insert_one(chat_info) + cache[chat_id] = chat_info + else: + mongo.chats_collection.update_one({"chat_id": chat_id}, {"$set": values}) + mongo_info = dict(mongo_info) + mongo_info.update(values) + cache[chat_id] = mongo_info + else: + cached_info.update(values) + mongo.chats_collection.update_one({"chat_id": chat_id}, {"$set": values})