initial

.deploy/deploy-dev.yaml

@@ -0,0 +1,24 @@
+version: "3.4"
+
+
+services:
+  certupdater:
+    image: mathwave/sprint-repo:certupdater
+    command: worker
+    environment:
+      MONGO_HOST: "mongo.develop.sprinthub.ru"
+      MONGO_PASSWORD: $MONGO_PASSWORD_DEV
+      STAGE: "development"
+    networks:
+      - configurator
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      update_config:
+        parallelism: 1
+        order: start-first
+
+networks:
+  configurator:
+    external: true

.deploy/deploy-prod.yaml

@@ -0,0 +1,45 @@
+version: "3.4"
+
+
+services:
+  worker:
+    image: mathwave/sprint-repo:pizda-bot
+    command: worker
+    environment:
+      MONGO_HOST: "mongo.sprinthub.ru"
+      MONGO_PASSWORD: $MONGO_PASSWORD_PROD
+      STAGE: "production"
+    networks:
+      - queues
+      - configurator
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      update_config:
+        parallelism: 1
+        order: start-first
+
+  pizda-bot-nginx:
+    image: mathwave/sprint-repo:pizda-bot
+    command: api
+    environment:
+      MONGO_HOST: "mongo.sprinthub.ru"
+      MONGO_PASSWORD: $MONGO_PASSWORD_PROD
+    networks:
+      - common-infra-nginx
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      update_config:
+        parallelism: 1
+        order: start-first
+
+networks:
+  common-infra-nginx:
+    external: true
+  queues:
+    external: true
+  configurator:
+    external: true

.gitea/workflows/deploy-dev.yaml

@@ -0,0 +1,43 @@
+name: Deploy Dev
+
+on:
+  pull_request:
+    branches:
+      - dev
+    types: [closed]
+
+jobs:
+  build:
+    name: Build
+    runs-on: [ dev ]
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: dev
+      - name: build
+        run: docker build -t mathwave/sprint-repo:pizda-bot .
+  push:
+    name: Push
+    runs-on: [ dev ]
+    needs: build
+    steps:
+      - name: push
+        run: docker push mathwave/sprint-repo:pizda-bot
+  deploy-dev:
+    name: Deploy dev
+    runs-on: [prod]
+    needs: push
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: dev
+      - name: deploy
+        env:
+          MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }}
+        run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml pizda-bot-development

.gitea/workflows/deploy-prod.yaml

@@ -0,0 +1,43 @@
+name: Deploy Prod
+
+on:
+  pull_request:
+    branches:
+      - prod
+    types: [closed]
+
+jobs:
+  build:
+    name: Build
+    runs-on: [ dev ]
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: prod
+      - name: build
+        run: docker build -t mathwave/sprint-repo:pizda-bot .
+  push:
+    name: Push
+    runs-on: [ dev ]
+    needs: build
+    steps:
+      - name: push
+        run: docker push mathwave/sprint-repo:pizda-bot
+  deploy-prod:
+    name: Deploy prod
+    runs-on: [prod]
+    needs: push
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+            ref: prod
+      - name: deploy
+        env:
+          MONGO_PASSWORD_PROD: ${{ secrets.MONGO_PASSWORD_PROD }}
+        run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-prod.yaml pizda-bot

.gitignore

@@ -0,0 +1,122 @@
+# Django #
+*.log
+*.pot
+*.pyc
+__pycache__
+db.sqlite3
+media
+data
+*/__pycache__
+
+# Backup files #
+*.bak
+
+# If you are using PyCharm #
+.idea
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/gradle.xml
+.idea/**/libraries
+*.iws /out/
+
+# Python #
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+.pytest_cache/
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+postgres-data
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery
+celerybeat-schedule.*
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.venv
+env/
+ENV/
+venv/
+env.bak/
+venv.bak/
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Sublime Text #
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+*.sublime-workspace
+*.sublime-project
+
+# sftp configuration file
+sftp-config.json
+
+# Package control specific files Package
+Control.last-run
+Control.ca-list
+Control.ca-bundle
+Control.system-ca-bundle
+GitHub.sublime-settings
+
+# Visual Studio Code #
+.vscode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history
+
+*pb2*
+.DS_Store

Dockerfile

@@ -0,0 +1,9 @@
+FROM docker:dind
+
+ENV PYTHONUNBUFFERED=1
+RUN apk add --update --no-cache python3 py3-pip && ln -sf python3 /usr/bin/python
+RUN python3 -m venv venv
+RUN venv/bin/python3 -m ensurepip
+RUN venv/bin/pip3 install --no-cache --upgrade pip setuptools
+
+ENTRYPOINT [ "venv/bin/python3", "main.py" ]

blob.py

@@ -0,0 +1,14 @@
+import os
+from minio import Minio
+
+MINIO_HOST = os.getenv("MINIO_HOST", "localhost") + ":9000"
+MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "serviceminioadmin")
+MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin")
+
+
+minio = Minio(
+    MINIO_HOST,
+    access_key=MINIO_ACCESS_KEY,
+    secret_key=MINIO_SECRET_KEY,
+    secure=False
+)

configurator.py

@@ -0,0 +1,88 @@
+import json
+import os
+import urllib.parse
+from threading import Thread
+from time import sleep
+
+from requests import get
+
+
+class ConfiguratorClient:
+    def __init__(self, app_name: str, stage: str, need_poll: bool = True):
+        self.app_name = app_name
+        self.stage = stage
+        self.endpoint = 'http://configurator/'
+        self.fetch_url = urllib.parse.urljoin(self.endpoint, '/api/v1/fetch')
+        self.config_storage = {}
+        self.experiment_storage = {}
+        self.staff_storage = {}
+        self.poll_data()
+        if need_poll:
+            self.poll_data_in_thread()
+
+    def poll_data_in_thread(self):
+        def inner():
+            while True:
+                sleep(30)
+                self.fetch()
+
+        Thread(target=inner, daemon=True).start()
+
+    def poll_data(self):
+        self.fetch(with_exception=True)
+
+    def request_with_retries(self, url, params, with_exception=False, retries_count=3):
+        exception_to_throw = None
+        for _ in range(retries_count):
+            try:
+                response = get(
+                    url,
+                    params=params
+                )
+                if response.status_code == 200:
+                    return response.json()
+                print(f'Failed to request {url}, status_code={response.status_code}')
+                exception_to_throw = Exception('Not 200 status')
+            except Exception as exc:
+                print(exc)
+                exception_to_throw = exc
+            sleep(1)
+        print(f'Failed fetching with retries: {url}, {params}')
+        if with_exception:
+            raise exception_to_throw
+
+    def fetch(self, with_exception=False):
+        if self.stage == 'local':
+            local_platform = json.loads(open('local_platform.json', 'r').read())
+            self.config_storage = local_platform['configs']
+            self.experiment_storage = local_platform['experiments']
+            self.staff_storage = {
+                key: set(value)
+                for key, value in local_platform['platform_staff'].items()
+            }
+            return
+        response_data = self.request_with_retries(self.fetch_url, {
+            'project': self.app_name,
+            'stage': self.stage,
+        }, with_exception)
+        self.config_storage = response_data['configs']
+        self.experiment_storage = response_data['experiments']
+        self.staff_storage = {
+            key: set(value)
+            for key, value in response_data['platform_staff'].items()
+        }
+
+    def is_staff(self, **kwargs):
+        for key, value in kwargs.items():
+            if value in self.staff_storage[key]:
+                return True
+        return False
+
+    def get_config(self, name):
+        return self.config_storage[name]
+
+    def get_experiment(self, name):
+        return self.experiment_storage[name]
+
+
+configurator = ConfiguratorClient("certupdater", os.getenv("STAGE"))

main.py

@@ -0,0 +1,44 @@
+import datetime
+import subprocess
+import time
+from configurator import configurator
+from mongo import mongo
+from blob import minio
+
+
+class Response:
+    code: int
+    out: str
+    err: str
+
+
+def call(command: str) -> Response:
+    p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
+    resp = p.wait()
+    response = Response()
+    response.code = resp
+    response.out, response.err = p.stdout.read().decode('utf-8'), p.stderr.read().decode('utf-8')
+    return response
+
+
+def get_hosts() -> list[str]:
+    return list(set(configurator.get_config("hosts") + ["platform.chocomarsh.com"]))
+
+
+def update_host(host: str):
+    gen_cert = call(f"docker exec $(docker ps -q -f name=infra_nginx) certbot --nginx --email emmtvv@gmail.com --agree-tos -d \"{host}\"")
+    if gen_cert.code != 0:
+        print("failed generating certificate")
+        return
+    
+
+
+while True:
+    now = datetime.datetime.now()
+    mongo_hosts = mongo.hosts
+    for host in get_hosts():
+        if now() + datetime.timedelta(days=14) > mongo_hosts[host]["expire_time"]:
+            update_host(host)
+            print(f"Host {host} updated")
+    minio.put_object("certupdater", "nginx.conf", )
+    time.sleep(5 * 60)

mongo.py

@@ -0,0 +1,41 @@
+import datetime
+import pymongo
+import os
+
+MONGO_USER = os.getenv("MONGO_USER", "mongo")
+MONGO_PASSWORD = os.getenv("MONGO_PASSWORD", "password")
+MONGO_HOST = os.getenv("MONGO_HOST", "localhost")
+
+
+class Mongo:
+    def __init__(self):
+        url = f"mongodb://{MONGO_USER}:{MONGO_PASSWORD}@{MONGO_HOST}:27017/"
+        self.client: pymongo.MongoClient = pymongo.MongoClient(url)
+        self.database = self.client.get_database("certupdater")
+        self.hosts_collection.create_index([
+            ("host", 1)
+        ])
+
+    def __getitem__(self, item):
+        return self.database.get_collection(item)
+
+    @property
+    def hosts_collection(self):
+        return self["hosts"]
+    
+    @property
+    def hosts(self):
+        hosts = {}
+        for host in self.hosts_collection.find({}):
+            hosts[host["host"]] = host
+        return hosts
+    
+    def update_date(self, host: str):
+        now = datetime.datetime.now()
+        if self.hosts_collection.find_one({"host": host}):
+            self.hosts_collection.update_one({"host": host}, {"$set": {"update_time": now, "expire_time": now + datetime.timedelta(days=90)}})
+        else:
+            self.hosts_collection.insert_one({"host": host, "update_time": now, "expire_time": now + datetime.timedelta(days=90)})
+
+
+mongo = Mongo()

storage.py

@@ -0,0 +1,42 @@
+from cachetools import TTLCache
+import os
+
+from utils.mongo import mongo
+
+CACHE_SIZE = int(os.getenv("CACHE_SIZE", 1000))
+CACHE_TTL = int(os.getenv("CACHE_TTL", 3600))
+
+cache = TTLCache(CACHE_SIZE, CACHE_TTL)
+
+
+def get_chat_info(chat_id: int) -> dict:
+    cached_info = cache.get(chat_id)
+    if cached_info is not None:
+        return cached_info
+    mongo_info = mongo.chats_collection.find_one({"chat_id": chat_id})
+    if mongo_info is not None:
+        cache[chat_id] = mongo_info
+        return mongo_info
+    chat_info = {"chat_id": chat_id, "state": "default", "probability": 100}
+    mongo.chats_collection.insert_one(chat_info)
+    cache[chat_id] = chat_info
+    return chat_info
+
+
+def set_values(chat_id: int, **values):
+    cached_info = cache.get(chat_id)
+    if cached_info is None:
+        mongo_info = mongo.chats_collection.find_one({"chat_id": chat_id})
+        if mongo_info is None:
+            chat_info = {"chat_id": chat_id, "state": "default", "probability": 100}
+            chat_info.update(values)
+            mongo.chats_collection.insert_one(chat_info)
+            cache[chat_id] = chat_info
+        else:
+            mongo.chats_collection.update_one({"chat_id": chat_id}, {"$set": values})
+            mongo_info = dict(mongo_info)
+            mongo_info.update(values)
+            cache[chat_id] = mongo_info
+    else:
+        cached_info.update(values)
+        mongo.chats_collection.update_one({"chat_id": chat_id}, {"$set": values})