initial

.deploy/deploy-dev.yaml

@@ -0,0 +1,22 @@
+version: "3.4"
+
+
+services:
+  queues-nginx:
+    image: mathwave/sprint-repo:queues
+    networks:
+      - common-infra-nginx
+    environment:
+      MONGO_HOST: "mongo.develop.sprinthub.ru"
+      MONGO_PASSWORD: $MONGO_PASSWORD_DEV
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      update_config:
+        parallelism: 1
+        order: start-first
+
+networks:
+  common-infra-nginx:
+    external: true

.deploy/deploy-prod.yaml

@@ -0,0 +1,22 @@
+version: "3.4"
+
+
+services:
+  queues-nginx:
+    image: mathwave/sprint-repo:queues
+    networks:
+      - common-infra-nginx
+    environment:
+      MONGO_HOST: "mongo.sprinthub.ru"
+      MONGO_PASSWORD: $MONGO_PASSWORD_PROD
+    deploy:
+      mode: replicated
+      restart_policy:
+        condition: any
+      update_config:
+        parallelism: 1
+        order: start-first
+
+networks:
+  common-infra-nginx:
+    external: true

.gitea/workflows/deploy-dev.yaml

@@ -0,0 +1,43 @@
+name: Deploy Dev
+
+on:
+  pull_request:
+    branches:
+      - dev
+    types: [closed]
+
+jobs:
+  build:
+    name: Build
+    runs-on: [ dev ]
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: dev
+      - name: build
+        run: docker build -t mathwave/sprint-repo:queues .
+  push:
+    name: Push
+    runs-on: [ dev ]
+    needs: build
+    steps:
+      - name: push
+        run: docker push mathwave/sprint-repo:queues
+  deploy-dev:
+    name: Deploy dev
+    runs-on: [dev]
+    needs: push
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: dev
+      - name: deploy
+        env:
+          MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }}
+        run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml infra

.gitea/workflows/deploy-prod.yaml

@@ -0,0 +1,43 @@
+name: Deploy Prod
+
+on:
+  pull_request:
+    branches:
+      - prod
+    types: [closed]
+
+jobs:
+  build:
+    name: Build
+    runs-on: [ dev ]
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: prod
+      - name: build
+        run: docker build -t mathwave/sprint-repo:queues .
+  push:
+    name: Push
+    runs-on: [ dev ]
+    needs: build
+    steps:
+      - name: push
+        run: docker push mathwave/sprint-repo:queues
+  deploy-prod:
+    name: Deploy prod
+    runs-on: [prod]
+    needs: push
+    steps:
+      - name: login
+        run: docker login -u mathwave -p ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+            ref: prod
+      - name: deploy
+        env:
+          MONGO_PASSWORD_PROD: ${{ secrets.MONGO_PASSWORD_PROD }}
+        run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-prod.yaml infra

.gitignore

@@ -0,0 +1,119 @@
+# Django #
+*.log
+*.pot
+*.pyc
+__pycache__
+db.sqlite3
+media
+data
+*/__pycache__
+
+# Backup files #
+*.bak
+
+# If you are using PyCharm #
+.idea
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/gradle.xml
+.idea/**/libraries
+*.iws /out/
+
+# Python #
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+.pytest_cache/
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+postgres-data
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery
+celerybeat-schedule.*
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.venv
+env/
+ENV/
+venv/
+env.bak/
+venv.bak/
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Sublime Text #
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+*.sublime-workspace
+*.sublime-project
+
+# sftp configuration file
+sftp-config.json
+
+# Package control specific files Package
+Control.last-run
+Control.ca-list
+Control.ca-bundle
+Control.system-ca-bundle
+GitHub.sublime-settings
+
+# Visual Studio Code #
+.vscode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.10
+
+RUN mkdir /usr/src/app
+WORKDIR /usr/src/app
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY . .
+
+ENTRYPOINT ["python", "main.py"]

app/routers/finish.py

@@ -0,0 +1,19 @@
+import bson
+import fastapi
+import pydantic
+
+from app.storage.mongo import tasks
+
+
+class RequestBody(pydantic.BaseModel):
+    id: str
+
+
+router = fastapi.APIRouter()
+
+
+@router.post('/api/v1/finish', status_code=fastapi.status.HTTP_202_ACCEPTED, responses={'404': {'description': 'Not found'}})
+async def execute(body: RequestBody):
+    if await tasks.finish_task(bson.ObjectId(body.id)):
+        return
+    raise fastapi.HTTPException(404)

app/routers/put.py

@@ -0,0 +1,30 @@
+import datetime
+import fastapi
+import pydantic
+import typing
+
+from app.storage.mongo import tasks
+from app.utils import time
+
+
+class RequestBody(pydantic.BaseModel):
+    payload: dict
+    seconds_to_execute: int
+    delay: int|None = None
+
+
+router = fastapi.APIRouter()
+
+
+@router.post('/api/v1/put', status_code=fastapi.status.HTTP_202_ACCEPTED)
+async def execute(body: RequestBody, queue: typing.Annotated[str, fastapi.Header()]):
+    now = time.now()
+    await tasks.add_task(
+        task=tasks.Task(
+            queue=queue, 
+            payload=body.payload, 
+            put_at=now, 
+            available_from=(now + datetime.timedelta(seconds=body.delay)) if body.delay else now,
+            seconds_to_execute=body.seconds_to_execute,
+        ),
+    )

app/routers/take.py

@@ -0,0 +1,22 @@
+import fastapi
+import pydantic
+import typing
+
+from app.storage.mongo import tasks
+
+
+router = fastapi.APIRouter()
+
+
+class Response(pydantic.BaseModel):
+    id: str
+    attempt: int
+    payload: dict
+
+
+@router.get('/api/v1/take', responses={404: {'description': 'Not found'}})
+async def execute(queue: typing.Annotated[str, fastapi.Header()]) -> Response:
+    task = await tasks.take_task(queue)
+    if not task:
+        raise fastapi.HTTPException(404)
+    return Response(id=str(task._id), attempt=task.attempts, payload=task.payload)

app/storage/mongo/__init__.py

@@ -0,0 +1,21 @@
+import os
+import motor
+import motor.motor_asyncio
+import pymongo
+
+
+MONGO_HOST = os.getenv('MONGO_HOST', 'localhost')
+MONGO_PASSWORD = os.getenv('MONGO_PASSWORD', 'password')
+
+CONNECTION_STRING = f'mongodb://mongo:{MONGO_PASSWORD}@{MONGO_HOST}:27017/'
+
+
+database: 'motor.MotorDatabase' = motor.motor_asyncio.AsyncIOMotorClient(CONNECTION_STRING).queues
+
+def create_indexes():
+    client = pymongo.MongoClient(CONNECTION_STRING)
+    database = client.get_database('queues')
+    database.get_collection('tasks').create_index([
+        ('queue', 1), 
+        ('available_from', 1),
+    ])

app/storage/mongo/tasks.py

@@ -0,0 +1,44 @@
+import asyncio
+import bson
+import datetime
+import pydantic
+import typing
+
+from app.storage.mongo import database
+from app.utils import time
+from bson import codec_options
+
+
+collection = database.get_collection("tasks", codec_options=codec_options.CodecOptions(tz_aware=True))
+
+
+class Task(pydantic.BaseModel):
+    queue: str
+    payload: dict
+    put_at: pydantic.AwareDatetime
+    available_from: pydantic.AwareDatetime
+    seconds_to_execute: int
+    _id: bson.ObjectId|None = None
+    taken_at: pydantic.AwareDatetime|None = None
+    attempts: int = 0
+
+
+async def add_task(task: Task) -> str:
+    result = await collection.insert_one(task.model_dump())
+    return result.inserted_id
+
+
+async def take_task(queue: str) -> typing.Optional[Task]:
+    now = time.now()
+    async for raw_task in collection.find({'queue': queue, 'available_from': {'$lte': now}}):
+        task = Task.model_validate(raw_task)
+        task._id = raw_task['_id']
+        if not task.taken_at or (task.taken_at + datetime.timedelta(seconds=task.seconds_to_execute)) < now:
+            await collection.update_one({'_id': task._id}, {'$set': {'taken_at': now, 'attempts': task.attempts + 1}})
+            return task
+    return None
+
+
+async def finish_task(id: bson.ObjectId) -> bool:
+    result = await collection.delete_one({'_id': id})
+    return result

app/utils/time.py

@@ -0,0 +1,4 @@
+import datetime
+
+
+now = lambda: datetime.datetime.now(datetime.UTC)

main.py

@@ -0,0 +1,21 @@
+import fastapi
+import uvicorn
+
+from app.storage import mongo
+
+from app.routers import take
+from app.routers import put
+from app.routers import finish
+
+
+app = fastapi.FastAPI()
+
+app.include_router(take.router)
+app.include_router(put.router)
+app.include_router(finish.router)
+
+mongo.create_indexes()
+
+
+if __name__ == '__main__':
+    uvicorn.run(app, host="0.0.0.0", port=1238)

requirements.txt

@@ -0,0 +1,20 @@
+annotated-types==0.7.0
+anyio==4.6.2.post1
+APScheduler==3.10.4
+click==8.1.7
+dnspython==2.7.0
+fastapi==0.115.4
+h11==0.14.0
+idna==3.10
+motor==3.6.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+pymongo==4.9.2
+pytz==2024.2
+redis==5.2.0
+six==1.16.0
+sniffio==1.3.1
+starlette==0.41.2
+typing_extensions==4.12.2
+tzlocal==5.2
+uvicorn==0.32.0