From be7bc1ef2c7bdcf47a1cdff57e14149e9f7cdca1 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Sat, 23 Nov 2024 22:20:12 +0300 Subject: [PATCH 1/7] update --- .deploy/deploy-dev.yaml | 6 ++---- .deploy/deploy-prod.yaml | 3 --- .gitea/workflows/deploy-dev.yaml | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index 53f7193..80e0af2 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -5,7 +5,7 @@ services: queues: image: mathwave/sprint-repo:queues networks: - - queues + - queues-development environment: MONGO_HOST: "mongo.develop.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_DEV @@ -18,7 +18,5 @@ services: order: start-first networks: - common-infra-nginx: - external: true - queues: + queues-development: external: true diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index 5c1962e..1e94afa 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -5,7 +5,6 @@ services: queues: image: mathwave/sprint-repo:queues networks: - - common-infra-nginx - queues environment: MONGO_HOST: "mongo.sprinthub.ru" @@ -19,7 +18,5 @@ services: order: start-first networks: - common-infra-nginx: - external: true queues: external: true diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml index 0150bdf..3efe3ac 100644 --- a/.gitea/workflows/deploy-dev.yaml +++ b/.gitea/workflows/deploy-dev.yaml @@ -28,7 +28,7 @@ jobs: run: docker push mathwave/sprint-repo:queues deploy-dev: name: Deploy dev - runs-on: [dev] + runs-on: [prod] needs: push steps: - name: login @@ -40,4 +40,4 @@ jobs: - name: deploy env: MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }} - run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml infra + run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml infra-development From a328d6feb272b2710b7ce675d4dfa84dbcab061b Mon Sep 17 00:00:00 2001 From: emmatveev Date: Mon, 25 Nov 2024 00:27:40 +0300 Subject: [PATCH 2/7] redis --- .deploy/deploy-dev.yaml | 2 ++ .deploy/deploy-prod.yaml | 2 ++ .gitea/workflows/deploy-dev.yaml | 1 + .gitea/workflows/deploy-prod.yaml | 1 + app/routers/take.py | 4 +++- app/storage/redis/__init__.py | 15 +++++++++++++++ app/storage/redis/lock.py | 8 ++++++++ 7 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 app/storage/redis/__init__.py create mode 100644 app/storage/redis/lock.py diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index 80e0af2..20e2223 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -9,6 +9,8 @@ services: environment: MONGO_HOST: "mongo.develop.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_DEV + REDIS_HOST: "redis.develop.sprinthub.ru" + REDIS_PASSWORD: $REDIS_PASSWORD_DEV deploy: mode: replicated restart_policy: diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index 1e94afa..b01f4c6 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -9,6 +9,8 @@ services: environment: MONGO_HOST: "mongo.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_PROD + REDIS_HOST: "redis.sprinthub.ru" + REDIS_PASSWORD: $REDIS_PASSWORD_PROD deploy: mode: replicated restart_policy: diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml index 3efe3ac..843315a 100644 --- a/.gitea/workflows/deploy-dev.yaml +++ b/.gitea/workflows/deploy-dev.yaml @@ -40,4 +40,5 @@ jobs: - name: deploy env: MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }} + REDIS_PASSWORD_DEV: ${{ secrets.REDIS_PASSWORD_DEV }} run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml infra-development diff --git a/.gitea/workflows/deploy-prod.yaml b/.gitea/workflows/deploy-prod.yaml index 9a6c82f..4ce27eb 100644 --- a/.gitea/workflows/deploy-prod.yaml +++ b/.gitea/workflows/deploy-prod.yaml @@ -40,4 +40,5 @@ jobs: - name: deploy env: MONGO_PASSWORD_PROD: ${{ secrets.MONGO_PASSWORD_PROD }} + REDIS_PASSWORD_PROD: ${{ secrets.REDIS_PASSWORD_PROD }} run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-prod.yaml infra diff --git a/app/routers/take.py b/app/routers/take.py index 9f8c8e4..a2abae1 100644 --- a/app/routers/take.py +++ b/app/routers/take.py @@ -3,6 +3,7 @@ import pydantic import typing from app.storage.mongo import tasks +from app.storage.redis import lock router = fastapi.APIRouter() @@ -16,7 +17,8 @@ class Response(pydantic.BaseModel): @router.get('/api/v1/take', responses={404: {'description': 'Not found'}}) async def execute(queue: typing.Annotated[str, fastapi.Header()]) -> Response: - task = await tasks.take_task(queue) + async with lock.acquire(queue): + task = await tasks.take_task(queue) if not task: raise fastapi.HTTPException(404) return Response(id=str(task._id), attempt=task.attempts, payload=task.payload) diff --git a/app/storage/redis/__init__.py b/app/storage/redis/__init__.py new file mode 100644 index 0000000..417b2f5 --- /dev/null +++ b/app/storage/redis/__init__.py @@ -0,0 +1,15 @@ +import os +import redis.asyncio + + +REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') +REDIS_PASSWORD = os.getenv('REDIS_PASSWORD') +if REDIS_PASSWORD: + URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:6379' +else: + URL = f'redis://{REDIS_HOST}:6379' + + + +pool = redis.asyncio.ConnectionPool.from_url(URL) +database = redis.Redis.from_pool(pool) diff --git a/app/storage/redis/lock.py b/app/storage/redis/lock.py new file mode 100644 index 0000000..5282a58 --- /dev/null +++ b/app/storage/redis/lock.py @@ -0,0 +1,8 @@ +import contextlib + +from app.storage import redis + + +@contextlib.contextmanager +def acquire(lock_name: str): + return redis.database.lock(lock_name) From 5efc884f12d722b6306414ce3aa1800e8fdb8fd1 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Mon, 25 Nov 2024 00:33:13 +0300 Subject: [PATCH 3/7] fix --- app/routers/take.py | 4 ++-- app/storage/mongo/tasks.py | 1 - app/storage/redis/__init__.py | 6 +++--- app/storage/redis/lock.py | 8 -------- 4 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 app/storage/redis/lock.py diff --git a/app/routers/take.py b/app/routers/take.py index a2abae1..34cddb4 100644 --- a/app/routers/take.py +++ b/app/routers/take.py @@ -3,7 +3,7 @@ import pydantic import typing from app.storage.mongo import tasks -from app.storage.redis import lock +from app.storage import redis router = fastapi.APIRouter() @@ -17,7 +17,7 @@ class Response(pydantic.BaseModel): @router.get('/api/v1/take', responses={404: {'description': 'Not found'}}) async def execute(queue: typing.Annotated[str, fastapi.Header()]) -> Response: - async with lock.acquire(queue): + async with redis.database.lock(queue): task = await tasks.take_task(queue) if not task: raise fastapi.HTTPException(404) diff --git a/app/storage/mongo/tasks.py b/app/storage/mongo/tasks.py index 0758760..e1b480a 100644 --- a/app/storage/mongo/tasks.py +++ b/app/storage/mongo/tasks.py @@ -1,4 +1,3 @@ -import asyncio import bson import datetime import pydantic diff --git a/app/storage/redis/__init__.py b/app/storage/redis/__init__.py index 417b2f5..3b59e59 100644 --- a/app/storage/redis/__init__.py +++ b/app/storage/redis/__init__.py @@ -1,5 +1,5 @@ import os -import redis.asyncio +import redis.asyncio as connection REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') @@ -11,5 +11,5 @@ else: -pool = redis.asyncio.ConnectionPool.from_url(URL) -database = redis.Redis.from_pool(pool) +pool = connection.from_url(URL) +database = connection.Redis.from_pool(pool) diff --git a/app/storage/redis/lock.py b/app/storage/redis/lock.py deleted file mode 100644 index 5282a58..0000000 --- a/app/storage/redis/lock.py +++ /dev/null @@ -1,8 +0,0 @@ -import contextlib - -from app.storage import redis - - -@contextlib.contextmanager -def acquire(lock_name: str): - return redis.database.lock(lock_name) From 11c30267381675c99248976483997c18d9fa086a Mon Sep 17 00:00:00 2001 From: emmatveev Date: Mon, 25 Nov 2024 00:49:29 +0300 Subject: [PATCH 4/7] fix --- app/storage/redis/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/app/storage/redis/__init__.py b/app/storage/redis/__init__.py index 3b59e59..fdffabb 100644 --- a/app/storage/redis/__init__.py +++ b/app/storage/redis/__init__.py @@ -4,12 +4,6 @@ import redis.asyncio as connection REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') REDIS_PASSWORD = os.getenv('REDIS_PASSWORD') -if REDIS_PASSWORD: - URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:6379' -else: - URL = f'redis://{REDIS_HOST}:6379' - -pool = connection.from_url(URL) -database = connection.Redis.from_pool(pool) +database = connection.Redis(host=REDIS_HOST, password=REDIS_PASSWORD) From 2ff6c22e0bd5c7e8b6a9756d3c94d75ca1ec7e76 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Mon, 25 Nov 2024 01:00:51 +0300 Subject: [PATCH 5/7] fix --- .deploy/deploy-dev.yaml | 2 -- .deploy/deploy-prod.yaml | 2 -- .gitea/workflows/deploy-dev.yaml | 1 - .gitea/workflows/deploy-prod.yaml | 1 - app/routers/take.py | 3 +-- app/storage/redis/__init__.py | 9 --------- 6 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 app/storage/redis/__init__.py diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index 20e2223..80e0af2 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -9,8 +9,6 @@ services: environment: MONGO_HOST: "mongo.develop.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_DEV - REDIS_HOST: "redis.develop.sprinthub.ru" - REDIS_PASSWORD: $REDIS_PASSWORD_DEV deploy: mode: replicated restart_policy: diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index b01f4c6..1e94afa 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -9,8 +9,6 @@ services: environment: MONGO_HOST: "mongo.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_PROD - REDIS_HOST: "redis.sprinthub.ru" - REDIS_PASSWORD: $REDIS_PASSWORD_PROD deploy: mode: replicated restart_policy: diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml index 843315a..3efe3ac 100644 --- a/.gitea/workflows/deploy-dev.yaml +++ b/.gitea/workflows/deploy-dev.yaml @@ -40,5 +40,4 @@ jobs: - name: deploy env: MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }} - REDIS_PASSWORD_DEV: ${{ secrets.REDIS_PASSWORD_DEV }} run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml infra-development diff --git a/.gitea/workflows/deploy-prod.yaml b/.gitea/workflows/deploy-prod.yaml index 4ce27eb..9a6c82f 100644 --- a/.gitea/workflows/deploy-prod.yaml +++ b/.gitea/workflows/deploy-prod.yaml @@ -40,5 +40,4 @@ jobs: - name: deploy env: MONGO_PASSWORD_PROD: ${{ secrets.MONGO_PASSWORD_PROD }} - REDIS_PASSWORD_PROD: ${{ secrets.REDIS_PASSWORD_PROD }} run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-prod.yaml infra diff --git a/app/routers/take.py b/app/routers/take.py index 34cddb4..f0cd112 100644 --- a/app/routers/take.py +++ b/app/routers/take.py @@ -17,8 +17,7 @@ class Response(pydantic.BaseModel): @router.get('/api/v1/take', responses={404: {'description': 'Not found'}}) async def execute(queue: typing.Annotated[str, fastapi.Header()]) -> Response: - async with redis.database.lock(queue): - task = await tasks.take_task(queue) + task = await tasks.take_task(queue) if not task: raise fastapi.HTTPException(404) return Response(id=str(task._id), attempt=task.attempts, payload=task.payload) diff --git a/app/storage/redis/__init__.py b/app/storage/redis/__init__.py deleted file mode 100644 index fdffabb..0000000 --- a/app/storage/redis/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -import redis.asyncio as connection - - -REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') -REDIS_PASSWORD = os.getenv('REDIS_PASSWORD') - - -database = connection.Redis(host=REDIS_HOST, password=REDIS_PASSWORD) From f35115d2f2e3d191a57978df6b71911569d25c11 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Wed, 27 Nov 2024 02:08:05 +0300 Subject: [PATCH 6/7] fix --- app/routers/take.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/routers/take.py b/app/routers/take.py index f0cd112..5085ce7 100644 --- a/app/routers/take.py +++ b/app/routers/take.py @@ -9,10 +9,17 @@ from app.storage import redis router = fastapi.APIRouter() +class Task(pydantic.BaseModel): + id: str + attempt: int + payload: dict + + class Response(pydantic.BaseModel): id: str attempt: int payload: dict + task: Task|None @router.get('/api/v1/take', responses={404: {'description': 'Not found'}}) @@ -20,4 +27,4 @@ async def execute(queue: typing.Annotated[str, fastapi.Header()]) -> Response: task = await tasks.take_task(queue) if not task: raise fastapi.HTTPException(404) - return Response(id=str(task._id), attempt=task.attempts, payload=task.payload) + return Response(id=str(task._id), attempt=task.attempts, payload=task.payload, task=Task(id=str(task._id), attempt=task.attempts, payload=task.payload)) From acfbb98e315b376521bb0a258b5e35fbb862d1b8 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Wed, 27 Nov 2024 02:11:24 +0300 Subject: [PATCH 7/7] fix --- app/routers/take.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/routers/take.py b/app/routers/take.py index 5085ce7..b56bd1a 100644 --- a/app/routers/take.py +++ b/app/routers/take.py @@ -3,7 +3,6 @@ import pydantic import typing from app.storage.mongo import tasks -from app.storage import redis router = fastapi.APIRouter()