From f4ca742f43c3a6bc2b9d4c6cd3817ca037693a96 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Fri, 22 Nov 2024 21:44:04 +0300 Subject: [PATCH] initial --- .deploy/deploy-dev.yaml | 22 ++++++ .deploy/deploy-prod.yaml | 22 ++++++ .gitea/workflows/deploy-dev.yaml | 43 +++++++++++ .gitea/workflows/deploy-prod.yaml | 43 +++++++++++ .gitignore | 119 ++++++++++++++++++++++++++++++ Dockerfile | 11 +++ app/__init__.py | 0 app/routers/__init__.py | 0 app/routers/configs.py | 42 +++++++++++ app/routers/experiments.py | 40 ++++++++++ app/routers/fetch.py | 57 ++++++++++++++ app/routers/staff.py | 43 +++++++++++ app/storage/__init__.py | 0 app/storage/mongo/__init__.py | 30 ++++++++ app/storage/mongo/configs.py | 38 ++++++++++ app/storage/mongo/experiments.py | 39 ++++++++++ app/storage/mongo/staff.py | 37 ++++++++++ app/utils/time.py | 4 + main.py | 23 ++++++ requirements.txt | 20 +++++ 20 files changed, 633 insertions(+) create mode 100644 .deploy/deploy-dev.yaml create mode 100644 .deploy/deploy-prod.yaml create mode 100644 .gitea/workflows/deploy-dev.yaml create mode 100644 .gitea/workflows/deploy-prod.yaml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/routers/__init__.py create mode 100644 app/routers/configs.py create mode 100644 app/routers/experiments.py create mode 100644 app/routers/fetch.py create mode 100644 app/routers/staff.py create mode 100644 app/storage/__init__.py create mode 100644 app/storage/mongo/__init__.py create mode 100644 app/storage/mongo/configs.py create mode 100644 app/storage/mongo/experiments.py create mode 100644 app/storage/mongo/staff.py create mode 100644 app/utils/time.py create mode 100644 main.py create mode 100644 requirements.txt diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml new file mode 100644 index 0000000..d58b2e5 --- /dev/null +++ b/.deploy/deploy-dev.yaml @@ -0,0 +1,22 @@ +version: "3.4" + + +services: + queues: + image: mathwave/sprint-repo:configurator + networks: + - configurator + environment: + MONGO_HOST: "mongo.develop.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_DEV + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + +networks: + configurator: + external: true diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml new file mode 100644 index 0000000..ebbe9c0 --- /dev/null +++ b/.deploy/deploy-prod.yaml @@ -0,0 +1,22 @@ +version: "3.4" + + +services: + queues: + image: mathwave/sprint-repo:configurator + networks: + - configurator + environment: + MONGO_HOST: "mongo.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_PROD + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + +networks: + configurator: + external: true diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml new file mode 100644 index 0000000..2631236 --- /dev/null +++ b/.gitea/workflows/deploy-dev.yaml @@ -0,0 +1,43 @@ +name: Deploy Dev + +on: + pull_request: + branches: + - dev + types: [closed] + +jobs: + build: + name: Build + runs-on: [ dev ] + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: dev + - name: build + run: docker build -t mathwave/sprint-repo:configurator . + push: + name: Push + runs-on: [ dev ] + needs: build + steps: + - name: push + run: docker push mathwave/sprint-repo:configurator + deploy-dev: + name: Deploy dev + runs-on: [dev] + needs: push + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: dev + - name: deploy + env: + MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }} + run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml infra diff --git a/.gitea/workflows/deploy-prod.yaml b/.gitea/workflows/deploy-prod.yaml new file mode 100644 index 0000000..29a07eb --- /dev/null +++ b/.gitea/workflows/deploy-prod.yaml @@ -0,0 +1,43 @@ +name: Deploy Prod + +on: + pull_request: + branches: + - prod + types: [closed] + +jobs: + build: + name: Build + runs-on: [ dev ] + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: prod + - name: build + run: docker build -t mathwave/sprint-repo:configurator . + push: + name: Push + runs-on: [ dev ] + needs: build + steps: + - name: push + run: docker push mathwave/sprint-repo:configurator + deploy-prod: + name: Deploy prod + runs-on: [prod] + needs: push + steps: + - name: login + run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }} + - name: checkout + uses: actions/checkout@v4 + with: + ref: prod + - name: deploy + env: + MONGO_PASSWORD_PROD: ${{ secrets.MONGO_PASSWORD_PROD }} + run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-prod.yaml infra diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..849d3ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,119 @@ +# Django # +*.log +*.pot +*.pyc +__pycache__ +db.sqlite3 +media +data +*/__pycache__ + +# Backup files # +*.bak + +# If you are using PyCharm # +.idea +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/gradle.xml +.idea/**/libraries +*.iws /out/ + +# Python # +*.py[cod] +*$py.class + +# Distribution / packaging +.Python build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +postgres-data + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +ENV/ +venv/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Sublime Text # +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# sftp configuration file +sftp-config.json + +# Package control specific files Package +Control.last-run +Control.ca-list +Control.ca-bundle +Control.system-ca-bundle +GitHub.sublime-settings + +# Visual Studio Code # +.vscode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..224e023 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12 + +RUN mkdir /usr/src/app +WORKDIR /usr/src/app + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +COPY . . + +ENTRYPOINT ["python", "main.py"] diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routers/configs.py b/app/routers/configs.py new file mode 100644 index 0000000..2467bdb --- /dev/null +++ b/app/routers/configs.py @@ -0,0 +1,42 @@ +import bson +import fastapi +import pydantic + +from app.storage.mongo import configs + + +class RequestPostBody(pydantic.BaseModel): + name: str + stage: str + project: str + + +class RequestPutBody(pydantic.BaseModel): + id: str + value: dict + + +class RequestDeleteBody(pydantic.BaseModel): + id: str + + +router = fastapi.APIRouter() + + +@router.post('/api/v1/configs', status_code=fastapi.status.HTTP_202_ACCEPTED) +async def post(body: RequestPostBody): + await configs.create(configs.Config(name=body.name, project=body.project, stage=body.stage, value={})) + + +@router.put('/api/v1/configs', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={404: {'description': 'Not found'}}) +async def put(body: RequestPutBody): + changed = await configs.update_data(id=bson.ObjectId(body.id), value=body.value) + if not changed: + raise fastapi.HTTPException(404) + + +@router.delete('/api/v1/configs', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={404: {'description': 'Not found'}}) +async def delete(body: RequestDeleteBody): + changed = await configs.delete(id=bson.ObjectId(body.id)) + if not changed: + raise fastapi.HTTPException(404) diff --git a/app/routers/experiments.py b/app/routers/experiments.py new file mode 100644 index 0000000..468e549 --- /dev/null +++ b/app/routers/experiments.py @@ -0,0 +1,40 @@ +import fastapi +import pydantic + +from app.storage.mongo import experiments + + +class RequestPostBody(pydantic.BaseModel): + name: str + stage: str + project: str + + +class RequestPutBody(pydantic.BaseModel): + name: str + stage: str + project: str + enabled: bool + condition: str + + +router = fastapi.APIRouter() + + +@router.post('/api/v1/experiments', status_code=fastapi.status.HTTP_202_ACCEPTED) +async def post(body: RequestPostBody): + await experiments.create(experiments.Experiment(name=body.name, project=body.project, stage=body.stage, enabled=False, condition='False')) + + +@router.put('/api/v1/experiments', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={404: {'description': 'Not found'}}) +async def put(body: RequestPutBody): + changed = await experiments.update(project=body.project, stage=body.stage, name=body.name, enabled=body.enabled, condition=body.condition) + if not changed: + raise fastapi.HTTPException(404) + + +@router.delete('/api/v1/experiments', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={404: {'description': 'Not found'}}) +async def delete(body: RequestPostBody): + changed = await experiments.delete(project=body.project, stage=body.stage, name=body.name) + if not changed: + raise fastapi.HTTPException(404) diff --git a/app/routers/fetch.py b/app/routers/fetch.py new file mode 100644 index 0000000..5ede8ef --- /dev/null +++ b/app/routers/fetch.py @@ -0,0 +1,57 @@ +import asyncio +import fastapi +import pydantic + +from app.storage.mongo import configs +from app.storage.mongo import experiments +from app.storage.mongo import staff + + +class ExperimentData(pydantic.BaseModel): + enabled: bool + condition: str + + +class PlatformStaff(pydantic.BaseModel): + vk_id: list[int] + yandex_id: list[int] + telegram_id: list[int] + email: list[str] + + +class ResponseBody(pydantic.BaseModel): + configs: dict[str, dict] + experiments: dict[str, ExperimentData] + platform_staff: PlatformStaff + + +router = fastapi.APIRouter() + + +@router.post('/api/v1/fetch') +async def execute(stage: str, project: str): + confs, exps, staffs = await asyncio.gather( + configs.get(project=project, stage=stage), + experiments.get(project=project, stage=stage), + staff.get(), + ) + platform_staff = PlatformStaff( + vk_id=[], + yandex_id=[], + telegram_id=[], + email=[], + ) + for user in staffs: + if user.vk_id: + platform_staff.vk_id.append(user.vk_id) + if user.yandex_id: + platform_staff.yandex_id.append(user.yandex_id) + if user.telegram_id: + platform_staff.telegram_id.append(user.telegram_id) + if user.email: + platform_staff.email.append(user.email) + return ResponseBody( + configs={conf.name: conf.value for conf in confs}, + experiments={exp.name: ExperimentData(enabled=exp.enabled, condition=exp.condition) for exp in exps}, + platform_staff=platform_staff, + ) diff --git a/app/routers/staff.py b/app/routers/staff.py new file mode 100644 index 0000000..bf382ec --- /dev/null +++ b/app/routers/staff.py @@ -0,0 +1,43 @@ +import fastapi +import pydantic + +from app.storage.mongo import staff + + +class RequestPutBody(pydantic.BaseModel): + platform_id: int + vk_id: int|None + yandex_id: int|None + telegram_id: int|None + email: str|None + + +class RequestPostBody(pydantic.BaseModel): + platform_id: int + email: str|None + + +class RequestDeleteBody(pydantic.BaseModel): + platform_id: int + + +router = fastapi.APIRouter() + + +@router.post('/api/v1/staff', status_code=fastapi.status.HTTP_202_ACCEPTED) +async def post(body: RequestPostBody): + await staff.create(staff=staff.Staff(platform_id=body.platform_id, email=body.email)) + + +@router.put('/api/v1/staff', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={404: {'description': 'Not found'}}) +async def put(body: RequestPutBody): + changed = await staff.update(platform_id=body.platform_id, email=body.email, vk_id=body.vk_id, yandex_id=body.yandex_id, telegram_id=body.telegram_id) + if not changed: + raise fastapi.HTTPException(404) + + +@router.delete('/api/v1/staff', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={404: {'description': 'Not found'}}) +async def delete(body: RequestDeleteBody): + changed = await staff.delete(platform_id=body.platform_id) + if not changed: + raise fastapi.HTTPException(404) diff --git a/app/storage/__init__.py b/app/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/storage/mongo/__init__.py b/app/storage/mongo/__init__.py new file mode 100644 index 0000000..e29b4e8 --- /dev/null +++ b/app/storage/mongo/__init__.py @@ -0,0 +1,30 @@ +import os +import motor +import motor.motor_asyncio +import pymongo + + +MONGO_HOST = os.getenv('MONGO_HOST', 'localhost') +MONGO_PASSWORD = os.getenv('MONGO_PASSWORD', 'password') + +CONNECTION_STRING = f'mongodb://mongo:{MONGO_PASSWORD}@{MONGO_HOST}:27017/' + + +database: 'motor.MotorDatabase' = motor.motor_asyncio.AsyncIOMotorClient(CONNECTION_STRING).configurator + +def create_indexes(): + client = pymongo.MongoClient(CONNECTION_STRING) + database = client.get_database('configurator') + database.get_collection('configs').create_index([ + ('stage', 1), + ('project', 1), + ('name', 1) + ]) + database.get_collection('experiments').create_index([ + ('stage', 1), + ('project', 1), + ('name', 1) + ]) + database.get_collection('staff').create_index([ + ('platform_id', 1), + ]) diff --git a/app/storage/mongo/configs.py b/app/storage/mongo/configs.py new file mode 100644 index 0000000..594a362 --- /dev/null +++ b/app/storage/mongo/configs.py @@ -0,0 +1,38 @@ +import bson +import pydantic + +from app.storage.mongo import database +from bson import codec_options + + +collection = database.get_collection("configs", codec_options=codec_options.CodecOptions(tz_aware=True)) + + +class Config(pydantic.BaseModel): + name: str + project: str + stage: str + value: dict + _id: bson.ObjectId|None = None + + +async def create(config: Config) -> str: + result = await collection.insert_one(config.model_dump()) + return result.inserted_id + + +async def update_data(project: str, stage: str, name: str, value: dict) -> bool: + result = await collection.update_one({'project': project, 'stage': stage, 'name': name}, {'$set': {'value': value}}) + return result.modified_count != 0 + + +async def delete(project: str, stage: str, name: str) -> bool: + result = await collection.delete_one({'project': project, 'stage': stage, 'name': name}) + return result.deleted_count != 0 + + +async def get(project: str, stage: str) -> list[Config]: + result = [] + async for item in collection.find({'stage': stage, 'project': project}): + result.append(Config.model_validate(item)) + return result diff --git a/app/storage/mongo/experiments.py b/app/storage/mongo/experiments.py new file mode 100644 index 0000000..0049a05 --- /dev/null +++ b/app/storage/mongo/experiments.py @@ -0,0 +1,39 @@ +import bson +import pydantic + +from app.storage.mongo import database +from bson import codec_options + + +collection = database.get_collection("experiments", codec_options=codec_options.CodecOptions(tz_aware=True)) + + +class Experiment(pydantic.BaseModel): + name: str + enabled: bool + condition: str + project: str + stage: str + _id: bson.ObjectId|None = None + + +async def create(experiment: Experiment) -> str: + result = await collection.insert_one(experiment.model_dump()) + return result.inserted_id + + +async def update(project: str, stage: str, name: str, enabled: bool, condition: str) -> bool: + result = await collection.update_one({'project': project, 'stage': stage, 'name': name}, {'$set': {'enabled': enabled, 'condition': condition}}) + return result.modified_count != 0 + + +async def delete(project: str, stage: str, name: str) -> bool: + result = await collection.delete_one({'project': project, 'stage': stage, 'name': name}) + return result.deleted_count != 0 + + +async def get(project: str, stage: str) -> list[Experiment]: + result = [] + async for item in collection.find({'stage': stage, 'project': project}): + result.append(Experiment.model_validate(item)) + return result diff --git a/app/storage/mongo/staff.py b/app/storage/mongo/staff.py new file mode 100644 index 0000000..ddc0a74 --- /dev/null +++ b/app/storage/mongo/staff.py @@ -0,0 +1,37 @@ +import pydantic + +from app.storage.mongo import database +from bson import codec_options + + +collection = database.get_collection("staff", codec_options=codec_options.CodecOptions(tz_aware=True)) + + +class Staff(pydantic.BaseModel): + platform_id: int + vk_id: int|None = None + yandex_id: int|None = None + telegram_id: int|None = None + email: str|None = None + + +async def create(staff: Staff) -> str: + result = await collection.insert_one(staff.model_dump()) + return result.inserted_id + + +async def update(platform_id: int, vk_id: int|None, yandex_id: int|None, telegram_id: int|None, email: str|None) -> bool: + result = await collection.update_one({'platform_id': platform_id}, {'$set': {'vk_id': vk_id, 'yandex_id': yandex_id, 'telegram_id': telegram_id, 'email': email}}) + return result.modified_count != 0 + + +async def delete(platform_id: int) -> bool: + result = await collection.delete_one({'platform_id': platform_id}) + return result.deleted_count != 0 + + +async def get() -> list[Staff]: + result = [] + async for item in collection.find({}): + result.append(Staff.model_validate(item)) + return result diff --git a/app/utils/time.py b/app/utils/time.py new file mode 100644 index 0000000..b51c065 --- /dev/null +++ b/app/utils/time.py @@ -0,0 +1,4 @@ +import datetime + + +now = lambda: datetime.datetime.now(datetime.UTC) diff --git a/main.py b/main.py new file mode 100644 index 0000000..2c2603e --- /dev/null +++ b/main.py @@ -0,0 +1,23 @@ +import fastapi +import uvicorn + +from app.routers import experiments +from app.routers import configs +from app.routers import staff +from app.routers import fetch + +from app.storage import mongo + + +app = fastapi.FastAPI() + +app.include_router(experiments.router) +app.include_router(configs.router) +app.include_router(staff.router) +app.include_router(fetch.router) + +mongo.create_indexes() + + +if __name__ == '__main__': + uvicorn.run(app, host="0.0.0.0", port=80) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..325371f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +annotated-types==0.7.0 +anyio==4.6.2.post1 +APScheduler==3.10.4 +click==8.1.7 +dnspython==2.7.0 +fastapi==0.115.4 +h11==0.14.0 +idna==3.10 +motor==3.6.0 +pydantic==2.9.2 +pydantic_core==2.23.4 +pymongo==4.9.2 +pytz==2024.2 +redis==5.2.0 +six==1.16.0 +sniffio==1.3.1 +starlette==0.41.2 +typing_extensions==4.12.2 +tzlocal==5.2 +uvicorn==0.32.0