From d629146375ee6c55a753a60ffd9fc41c0bea931e Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Mon, 14 Sep 2026 19:00:25 -0230 Subject: [PATCH 01/16] 830 Updated database schema and added migration --- .../dd4abb2b6541_optimize_item_table.py | 495 ++++++++++++++++++ src/backend_api/app/core/models/models.py | 115 ++-- 2 files changed, 570 insertions(+), 40 deletions(-) create mode 100644 src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py diff --git a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py new file mode 100644 index 00000000..f8bdbee4 --- /dev/null +++ b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py @@ -0,0 +1,495 @@ +"""optimize item table + +Revision ID: dd4abb2b6541 +Revises: 965e766db0a0 +Create Date: 2026-07-23 15:20:18.565656 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "dd4abb2b6541" +down_revision: Union[str, None] = "965e766db0a0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "_temp_item_availability", + sa.Column("temp_game_item_id", sa.Text(), nullable=False), # Temporary column + sa.Column( + "temp_league_id", sa.SmallInteger(), nullable=False + ), # Temporary column + # sa.Column("itemId", sa.Integer(), nullable=False), # Will be added later + sa.Column("currencyId", sa.Integer(), nullable=False), + sa.Column("currencyAmount", sa.Float(4), nullable=False), + sa.Column("isAsync", sa.Boolean()), + sa.Column("validFrom", sa.SmallInteger(), nullable=False), + sa.Column("validTo", sa.SmallInteger()), + sa.ForeignKeyConstraint( + ["currencyId"], + ["currency.currencyId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ), + ) + + op.execute(""" + INSERT INTO _temp_item_availability ( + temp_game_item_id, + temp_league_id, + "currencyId", + "currencyAmount", + "validFrom", + "validTo" + ) + WITH ordered AS ( + SELECT + i.*, + LAG(i."createdHoursSinceLaunch") OVER w AS prev_hour, + LAG(i."leagueId") OVER w AS prev_league, + LAG(i."currencyId") OVER w AS prev_currency, + LAG(i."currencyAmount") OVER w AS prev_amount + FROM item i + WINDOW w AS ( + PARTITION BY i."gameItemId" + ORDER BY i."createdHoursSinceLaunch" + ) + ), + marked AS ( + SELECT + *, + CASE + WHEN + prev_hour IS NULL + OR o."createdHoursSinceLaunch" > prev_hour + 1 + OR o."leagueId" IS DISTINCT FROM prev_league + OR o."currencyId" IS DISTINCT FROM prev_currency + OR o."currencyAmount" IS DISTINCT FROM prev_amount + THEN 1 + ELSE 0 + END AS new_listing + FROM ordered o + ), + grouped AS ( + SELECT + *, + SUM(new_listing) OVER ( + PARTITION BY m."gameItemId" + ORDER BY m."createdHoursSinceLaunch" + ) AS grp + FROM marked m + ), + collapsed AS ( + SELECT + g."gameItemId", + g."leagueId", + g."currencyId", + g."currencyAmount", + grp, + MIN(g."createdHoursSinceLaunch") AS "validFrom", + MAX(g."createdHoursSinceLaunch") + 1 AS observedValidTo + FROM grouped g + GROUP BY + g."gameItemId", + g."leagueId", + g."currencyId", + g."currencyAmount", + grp + ) + SELECT + c."gameItemId", + c."leagueId", + c."currencyId", + c."currencyAmount", + c."validFrom", + CASE + WHEN LEAD(grp) OVER ( + PARTITION BY c."gameItemId" + ORDER BY grp + ) IS NULL + THEN c."validFrom" + 1 + ELSE observedValidTo + END AS "validTo" + + FROM collapsed c; + """) + # TODO optimize when writing the plotting query + # sa.Index("ix_item_availability_itemId_validFrom", "itemId", "validFrom"), + + with op.batch_alter_table("item") as batch_op: + batch_op.drop_column("prefixes") + batch_op.drop_column("suffixes") + batch_op.drop_column("currencyAmount") + batch_op.drop_column("currencyId") + batch_op.execute(""" + DELETE FROM item i + WHERE i."itemId" NOT IN ( + SELECT MIN(i2."itemId") + FROM item i2 + GROUP BY i2."gameItemId", i2."leagueId" + ) + """) + batch_op.execute(""" + DELETE FROM item i + WHERE i."itemId" NOT IN ( + SELECT im."itemId" + FROM item_modifier im + ) + """) + batch_op.alter_column("itemId", new_column_name="old_item_id") + + # Rebuilds the item table to remove the hypertable. + op.execute(""" + CREATE TABLE _item_temp AS + SELECT DISTINCT ON ("gameItemId", "leagueId") + ROW_NUMBER() OVER (ORDER BY "gameItemId", "leagueId") AS "itemId", + * + FROM item i + ORDER BY + "gameItemId", + "leagueId"; + + DROP TABLE item; + ALTER TABLE _item_temp + ALTER COLUMN "itemId" SET NOT NULL; + ALTER TABLE _item_temp + ALTER COLUMN "itemId" ADD GENERATED ALWAYS AS IDENTITY; + SELECT setval( + pg_get_serial_sequence('_item_temp', 'itemId'), + COALESCE((SELECT MAX("itemId") FROM _item_temp), 1), + TRUE + ); + + ALTER TABLE _item_temp RENAME TO item; + """) + + with op.batch_alter_table("item") as batch_op: + batch_op.drop_column("delve") + batch_op.drop_column("foilVariation") + batch_op.create_primary_key("item_pkey", ["itemId"]) + batch_op.create_foreign_key( + "fk_item_league", + "league", + ["leagueId"], + ["leagueId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + batch_op.create_foreign_key( + "fk_item_item_base_type", + "item_base_type", + ["itemBaseTypeId"], + ["itemBaseTypeId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + batch_op.alter_column("gameItemId", nullable=False) + batch_op.alter_column("leagueId", nullable=False) + batch_op.alter_column( + "createdHoursSinceLaunch", new_column_name="firstObserved", nullable=False + ) + batch_op.alter_column("name", nullable=False) + batch_op.alter_column("itemBaseTypeId", nullable=False) + batch_op.alter_column("rarity", nullable=False) + batch_op.alter_column("ilvl", nullable=False) + batch_op.alter_column("identified", nullable=False) + + # TODO update when writing the plotting query + batch_op.create_index( + "ix_item_leagueId_itemBaseTypeId", + ["leagueId", "itemBaseTypeId", "firstObserved"], + ) + + op.create_table( + "item_availability", + sa.Column( + "availabilityId", + sa.Integer(), + sa.Identity(always=False, start=1, increment=1), + nullable=False, + primary_key=True, + ), + sa.Column("itemId", sa.Integer(), nullable=False), + sa.Column("currencyId", sa.Integer(), nullable=False), + sa.Column("currencyAmount", sa.Float(4), nullable=False), + sa.Column("isAsync", sa.Boolean()), + sa.Column("validFrom", sa.SmallInteger(), nullable=False), + sa.Column("validTo", sa.SmallInteger()), + sa.ForeignKeyConstraint( + ["currencyId"], + ["currency.currencyId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ), + sa.ForeignKeyConstraint( + ["itemId"], + ["item.itemId"], + ondelete="CASCADE", + onupdate="CASCADE", + ), + # TODO optimize when writing the plotting query + sa.Index("ix_item_availability_itemId_validFrom", "itemId", "validFrom"), + ) + op.execute(""" + INSERT INTO item_availability ( + "itemId", + "currencyId", + "currencyAmount", + "validFrom", + "validTo" + ) + SELECT + i."itemId", + ia."currencyId", + ia."currencyAmount", + ia."validFrom", + ia."validTo" + FROM item i + LEFT JOIN _temp_item_availability ia + ON i."gameItemId" = ia.temp_game_item_id AND i."leagueId" = ia.temp_league_id; + """) + + op.drop_table("_temp_item_availability") + + # Rebuilds the item_modifier table to use the new itemId, and to remove the hypertable. + op.execute(""" + CREATE TABLE _item_modifier_temp AS + SELECT + i."itemId"::INT, + im."modifierId", + im.position, + im.roll + FROM item i + LEFT JOIN item_modifier im + ON i.old_item_id = im."itemId"; + + DROP TABLE item_modifier; + + ALTER TABLE _item_modifier_temp RENAME TO item_modifier; + """) + op.drop_column("item", "old_item_id") + + with op.batch_alter_table("item_modifier") as batch_op: + batch_op.alter_column("itemId", nullable=False) + batch_op.alter_column("modifierId", nullable=False) + batch_op.alter_column("position", nullable=False) + batch_op.alter_column( + "roll", existing_type=sa.Float(), type_=sa.Float(4), nullable=True + ) + batch_op.create_primary_key( + "item_modifier_pkey", ["itemId", "modifierId", "position"] + ) + batch_op.create_foreign_key( + "fk_item_modifier_item", + "item", + ["itemId"], + ["itemId"], + ondelete="CASCADE", + onupdate="CASCADE", + ) + batch_op.create_foreign_key( + "fk_item_modifier_modifier", + "modifier", + ["modifierId", "position"], + ["modifierId", "position"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + + +def downgrade() -> None: + raise NotImplementedError("Downgrade is not supported for this migration") + # Remove the FK before restoring the old item schema. + # op.drop_constraint( + # "fk_item_availability_item", + # "item_availability", + # type_="foreignkey", + # ) + # # Restore the columns that were removed by upgrade(). + # with op.batch_alter_table("item") as batch_op: + # batch_op.drop_constraint( + # "item_pkey", + # type_="primary", + # ) + + # batch_op.alter_column( + # "itemId", + # new_column_name="gameItemId", + # ) + + # batch_op.add_column( + # sa.Column( + # "itemId", + # sa.Integer(), + # sa.Identity(always=False, start=1, increment=1), + # nullable=False, + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "prefixes", + # sa.Integer(), + # nullable=False, + # server_default="0", + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "suffixes", + # sa.Integer(), + # nullable=False, + # server_default="0", + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "currencyId", + # sa.Integer(), + # nullable=True, + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "currencyAmount", + # sa.Float(4), + # nullable=True, + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "createdHoursSinceLaunch", + # sa.Integer(), + # nullable=True, + # ) + # ) # Build the complete historical item set from item_availability. + # # + # # The upgraded table still contains one representative item row for + # # every (gameItemId, leagueId). We use that row as the template for + # # all of the columns that were not part of item_availability. + # # + # # validFrom is inclusive and validTo is exclusive. + # op.execute(""" + # CREATE TEMP TABLE _item_template + # ON COMMIT DROP + # AS + # SELECT DISTINCT ON ("gameItemId", "leagueId") + # "gameItemId", + # "foilVariation", + # "identified", + # "corrupted", + # "delve", + # "fractured", + # "synthesised", + # "replica", + # "searing", + # "tangled", + # "influences", + # "name", + # "itemBaseTypeId", + # "ilvl", + # "rarity", + # "leagueId" + # FROM item + # ORDER BY + # "gameItemId", + # "leagueId", + # "itemId"; + # """) + + # # Remove the representative rows created by the upgrade. They are + # # replaced by the reconstructed hourly history below. + # # + # # No metadata is lost because it has already been copied into the + # # temporary template table. + # op.execute(""" + # DELETE FROM item; + # """) + + # # Reconstruct one row per hour. + # # + # # item_availability contains intervals: + # # + # # [validFrom, validTo) + # # + # # so generate_series(validFrom, validTo - 1) gives exactly the + # # original hourly rows represented by each interval. + # # + # # A new itemId is generated by the identity column. + # op.execute(""" + # INSERT INTO item ( + # "gameItemId", + # "prefixes", + # "suffixes", + # "foilVariation", + # "identified", + # "corrupted", + # "delve", + # "fractured", + # "synthesised", + # "replica", + # "searing", + # "tangled", + # "influences", + # "name", + # "itemBaseTypeId", + # "createdHoursSinceLaunch", + # "leagueId", + # "currencyId", + # "ilvl", + # "currencyAmount", + # "rarity" + # ) + # SELECT + # ia."itemId" AS "gameItemId", + # 0 AS "prefixes", + # 0 AS "suffixes", + # t."foilVariation", + # t."identified", + # t."corrupted", + # t."delve", + # t."fractured", + # t."synthesised", + # t."replica", + # t."searing", + # t."tangled", + # t."influences", + # t."name", + # t."itemBaseTypeId", + # h."createdHoursSinceLaunch", + # ia."leagueId", + # ia."currencyId", + # t."ilvl", + # ia."currencyAmount", + # t."rarity" + # FROM item_availability ia + # JOIN _item_template t + # ON t."gameItemId" = ia."itemId" + # AND t."leagueId" = ia."leagueId" + # CROSS JOIN LATERAL generate_series( + # ia."validFrom", + # ia."validTo" - 1 + # ) AS h("createdHoursSinceLaunch") + # ORDER BY + # ia."itemId", + # ia."leagueId", + # h."createdHoursSinceLaunch"; + # """) + + # # Restore the original identity primary key. + # with op.batch_alter_table("item") as batch_op: + # batch_op.create_primary_key( + # "item_pkey", + # ["itemId"], + # ) + # op.drop_table("item_availability") diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index bbe6f325..3e5188f2 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -2,7 +2,6 @@ from datetime import datetime from sqlalchemy import ( - BigInteger, Boolean, CheckConstraint, DateTime, @@ -68,6 +67,74 @@ class ItemBaseType(Base): relatedUniques: Mapped[str | None] = mapped_column(Text) +class ItemAvailability(Base): + __tablename__ = "item_availability" + + availabilityId: Mapped[int] = mapped_column( + Integer, primary_key=True, autoincrement=True + ) + itemId: Mapped[str] = mapped_column( + Text, + ForeignKey("item_temp.itemId", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + ) + currencyId: Mapped[int] = mapped_column( + Integer, + ForeignKey("currency.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), + nullable=False, + ) + currencyAmount: Mapped[float] = mapped_column(Float(4), nullable=False) + isAsync: Mapped[bool] = mapped_column(Boolean, nullable=False) + validFrom: Mapped[int] = mapped_column(SmallInteger, nullable=False) + validTo: Mapped[int | None] = mapped_column(SmallInteger, nullable=True) + + __table_args__ = (Index("ix_item_id_valid_from", "itemId", "validFrom"),) + + +class Item(Base): + __tablename__ = "item" + itemId: Mapped[int] = mapped_column( + Integer, + Identity(start=1, increment=1), + primary_key=True, + ) + gameItemId: Mapped[str | None] = mapped_column(Text, nullable=False) + leagueId: Mapped[SmallInteger] = mapped_column( + SmallInteger, + ForeignKey("league.leagueId", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + ) + firstObserved: Mapped[int] = mapped_column(SmallInteger, nullable=False) + name: Mapped[str | None] = mapped_column(Text, nullable=False) + itemBaseTypeId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey( + "item_base_type.itemBaseTypeId", ondelete="RESTRICT", onupdate="CASCADE" + ), + nullable=False, + ) + ilvl: Mapped[int] = mapped_column(SmallInteger, nullable=False) + rarity: Mapped[str] = mapped_column(Text, nullable=False) + + identified: Mapped[bool] = mapped_column(Boolean, nullable=False) + corrupted: Mapped[bool | None] = mapped_column(Boolean) + + replica: Mapped[bool | None] = mapped_column(Boolean) + searing: Mapped[bool | None] = mapped_column(Boolean) + tangled: Mapped[bool | None] = mapped_column(Boolean) + influences: Mapped[dict[str, str] | None] = mapped_column( + JSONB + ) # elder, shaper, warlord etc + + __table_args__ = ( + Index( + "ix_item_leagueId_itemBaseTypeId", + "leagueId", + "itemBaseTypeId", + ), + ) + + class _ItemBase: name: Mapped[str | None] = mapped_column(Text, nullable=False) itemBaseTypeId: Mapped[int] = mapped_column( @@ -84,10 +151,11 @@ class _ItemBase: nullable=False, ) itemId: Mapped[int] = mapped_column( - BigInteger, + Integer, Identity(start=1, increment=1, always=True), primary_key=True, # Primary key constraint gets removed on hypertable creation ) + currencyId: Mapped[int] = mapped_column( Integer, ForeignKey("currency.currencyId", ondelete="RESTRICT"), @@ -99,38 +167,6 @@ class _ItemBase: rarity: Mapped[str] = mapped_column(Text, nullable=False) -class Item(_ItemBase, Base): - # Hypertable - # For hypertable specs, see alembic revision `cc29b89156db' - __tablename__ = "item" - # TODO do something about None and make it not nullable - gameItemId: Mapped[str | None] = mapped_column(Text) - prefixes: Mapped[int | None] = mapped_column(SmallInteger) - suffixes: Mapped[int | None] = mapped_column(SmallInteger) - foilVariation: Mapped[int | None] = mapped_column(SmallInteger) - identified: Mapped[bool] = mapped_column(Boolean, nullable=False) - corrupted: Mapped[bool | None] = mapped_column(Boolean) - delve: Mapped[bool | None] = mapped_column(Boolean) - fractured: Mapped[bool | None] = mapped_column(Boolean) - synthesised: Mapped[bool | None] = mapped_column(Boolean) - replica: Mapped[bool | None] = mapped_column(Boolean) - searing: Mapped[bool | None] = mapped_column(Boolean) - tangled: Mapped[bool | None] = mapped_column(Boolean) - influences: Mapped[dict[str, str] | None] = mapped_column( - JSONB - ) # elder, shaper, warlord etc - - __table_args__ = ( - Index( - "ix_item_name_itemBaseTypeId_createdHoursSinceLaunch_leagueId", - "name", - "itemBaseTypeId", - "createdHoursSinceLaunch", - "leagueId", - ), - ) - - class UnidentifiedItem(_ItemBase, Base): """ IS-A item relation, couldn't be bothered finding the actual way to implement it. @@ -244,6 +280,10 @@ class ItemModifier(Base): # For hypertable specs, see alembic revision `cc29b89156db' __tablename__ = "item_modifier" + itemId: Mapped[int] = mapped_column( + Integer, + nullable=False, + ) modifierId: Mapped[int] = mapped_column( SmallInteger, @@ -253,16 +293,11 @@ class ItemModifier(Base): SmallInteger, nullable=False, ) - createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) - itemId: Mapped[int] = mapped_column( - BigInteger, - nullable=False, - primary_key=True, # Primary key constraint gets removed on hypertable creation - ) roll: Mapped[float | None] = mapped_column( Float(4), ) __table_args__ = ( + PrimaryKeyConstraint("itemId", "modifierId", "position"), ForeignKeyConstraint( ["modifierId", "position"], ["modifier.modifierId", "modifier.position"], From 5aba49d2ae5659c4077a5f6aded235390f6b924d Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Mon, 14 Sep 2026 19:18:09 -0230 Subject: [PATCH 02/16] #830 Added basic api functionality for item_availability. Still more work to be done in this regard --- src/backend_api/app/api/api.py | 7 ++++ src/backend_api/app/api/routes/__init__.py | 1 + .../app/api/routes/item_availability.py | 39 ++++++++++++++++++ src/backend_api/app/core/models/models.py | 9 ++++- src/backend_api/app/core/schemas/__init__.py | 6 +++ src/backend_api/app/core/schemas/item.py | 18 ++++----- .../app/core/schemas/item_availability.py | 40 +++++++++++++++++++ .../app/core/schemas/item_modifier.py | 4 +- src/backend_api/app/crud/__init__.py | 17 ++++++++ 9 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 src/backend_api/app/api/routes/item_availability.py create mode 100644 src/backend_api/app/core/schemas/item_availability.py diff --git a/src/backend_api/app/api/api.py b/src/backend_api/app/api/api.py index 19175c6a..545d49a7 100644 --- a/src/backend_api/app/api/api.py +++ b/src/backend_api/app/api/api.py @@ -4,6 +4,8 @@ currency, currency_prefix, item, + item_availability, + item_availability_prefix, item_base_type, item_base_type_prefix, item_modifier, @@ -31,6 +33,11 @@ api_router.include_router( currency.router, prefix=f"/{currency_prefix}", tags=[currency_prefix] ) +api_router.include_router( + item_availability.router, + prefix=f"/{item_availability_prefix}", + tags=[item_availability_prefix], +) api_router.include_router( item_base_type.router, prefix=f"/{item_base_type_prefix}", diff --git a/src/backend_api/app/api/routes/__init__.py b/src/backend_api/app/api/routes/__init__.py index 8cc7f8c0..2f62f675 100644 --- a/src/backend_api/app/api/routes/__init__.py +++ b/src/backend_api/app/api/routes/__init__.py @@ -1,4 +1,5 @@ from app.api.routes.currency import currency_prefix +from app.api.routes.item_availability import item_availability_prefix from app.api.routes.item_base_type import item_base_type_prefix from app.api.routes.item_modifier import item_modifier_prefix from app.api.routes.item import item_prefix diff --git a/src/backend_api/app/api/routes/item_availability.py b/src/backend_api/app/api/routes/item_availability.py new file mode 100644 index 00000000..2a0eb33d --- /dev/null +++ b/src/backend_api/app/api/routes/item_availability.py @@ -0,0 +1,39 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +import app.core.schemas as schemas +from app.api.deps import ( + get_current_active_superuser, + get_db, +) +from app.api.params import FilterParams +from app.crud import CRUD_itemAvailability + +router = APIRouter() + + +item_availability_prefix = "itemAvailability" + + +@router.get( + "/", + response_model=schemas.ItemAvailability | list[schemas.ItemAvailability], + dependencies=[Depends(get_current_active_superuser)], +) +async def get_all_item_modifiers( + filter_params: Annotated[FilterParams, Query()], + db: Session = Depends(get_db), +): + """ + Get all available items. + + Returns a list of all available items. + """ + + all_item_availability = await CRUD_itemAvailability.get( + db=db, filter_params=filter_params + ) + + return all_item_availability diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index 3e5188f2..4644b80c 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -78,16 +78,19 @@ class ItemAvailability(Base): ForeignKey("item_temp.itemId", ondelete="CASCADE", onupdate="CASCADE"), nullable=False, ) + currencyId: Mapped[int] = mapped_column( Integer, ForeignKey("currency.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), nullable=False, ) currencyAmount: Mapped[float] = mapped_column(Float(4), nullable=False) - isAsync: Mapped[bool] = mapped_column(Boolean, nullable=False) + validFrom: Mapped[int] = mapped_column(SmallInteger, nullable=False) validTo: Mapped[int | None] = mapped_column(SmallInteger, nullable=True) + isAsync: Mapped[bool | None] = mapped_column(Boolean, nullable=False) + __table_args__ = (Index("ix_item_id_valid_from", "itemId", "validFrom"),) @@ -105,7 +108,7 @@ class Item(Base): nullable=False, ) firstObserved: Mapped[int] = mapped_column(SmallInteger, nullable=False) - name: Mapped[str | None] = mapped_column(Text, nullable=False) + name: Mapped[str] = mapped_column(Text, nullable=False) itemBaseTypeId: Mapped[int] = mapped_column( SmallInteger, ForeignKey( @@ -119,6 +122,8 @@ class Item(Base): identified: Mapped[bool] = mapped_column(Boolean, nullable=False) corrupted: Mapped[bool | None] = mapped_column(Boolean) + fractured: Mapped[bool | None] = mapped_column(Boolean) + synthesised: Mapped[bool | None] = mapped_column(Boolean) replica: Mapped[bool | None] = mapped_column(Boolean) searing: Mapped[bool | None] = mapped_column(Boolean) tangled: Mapped[bool | None] = mapped_column(Boolean) diff --git a/src/backend_api/app/core/schemas/__init__.py b/src/backend_api/app/core/schemas/__init__.py index 4d69acff..5df53cdc 100644 --- a/src/backend_api/app/core/schemas/__init__.py +++ b/src/backend_api/app/core/schemas/__init__.py @@ -2,6 +2,12 @@ from .currency import Currency, CurrencyCreate, CurrencyInDB, CurrencyUpdate from .item import Item, ItemCreate, ItemInDB, ItemUpdate +from .item_availability import ( + ItemAvailability, + ItemAvailabilityCreate, + ItemAvailabilityInDB, + ItemAvailabilityUpdate, +) from .item_base_type import ( ItemBaseType, ItemBaseTypeCreate, diff --git a/src/backend_api/app/core/schemas/item.py b/src/backend_api/app/core/schemas/item.py index c42c04a3..40421c98 100644 --- a/src/backend_api/app/core/schemas/item.py +++ b/src/backend_api/app/core/schemas/item.py @@ -14,31 +14,30 @@ class Influences(_pydantic.BaseModel): class _BaseItem(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) - name: str | None = None + gameItemId: str leagueId: int + + firstObserved: int + + name: str itemBaseTypeId: int ilvl: int rarity: str - gameItemId: str | None = None + identified: bool = True - currencyAmount: float | None = None - currencyId: int | None = None corrupted: bool | None = None - delve: bool | None = None + fractured: bool | None = None synthesised: bool | None = None replica: bool | None = None influences: Influences | None = None searing: bool | None = None tangled: bool | None = None - prefixes: int | None = None - suffixes: int | None = None - foilVariation: int | None = None # Properties to receive on item creation class ItemCreate(_BaseItem): - createdHoursSinceLaunch: int + pass # Properties to receive on update @@ -48,7 +47,6 @@ class ItemUpdate(_BaseItem): # Properties shared by models stored in DB class ItemInDBBase(_BaseItem): - createdHoursSinceLaunch: int itemId: int diff --git a/src/backend_api/app/core/schemas/item_availability.py b/src/backend_api/app/core/schemas/item_availability.py new file mode 100644 index 00000000..db323e3c --- /dev/null +++ b/src/backend_api/app/core/schemas/item_availability.py @@ -0,0 +1,40 @@ +import pydantic as _pydantic + + +# Shared item props +class _BaseItemAvailability(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + itemId: int + + currencyId: int + currencyAmount: float + + validFrom: int + validTo: int | None = None + + isAsync: bool | None = None + + +# Properties to receive on item creation +class ItemAvailabilityCreate(_BaseItemAvailability): + pass + + +# Properties to receive on update +class ItemAvailabilityUpdate(_BaseItemAvailability): + pass + + +# Properties shared by models stored in DB +class ItemAvailabilityInDBBase(_BaseItemAvailability): + availabilityId: int + + +# Properties to return to client +class ItemAvailability(ItemAvailabilityInDBBase): + pass + + +# Properties stored in DB +class ItemAvailabilityInDB(ItemAvailabilityInDBBase): + pass diff --git a/src/backend_api/app/core/schemas/item_modifier.py b/src/backend_api/app/core/schemas/item_modifier.py index 953f9a1c..e04a9ba5 100644 --- a/src/backend_api/app/core/schemas/item_modifier.py +++ b/src/backend_api/app/core/schemas/item_modifier.py @@ -13,7 +13,7 @@ class _BaseItemModifier(_pydantic.BaseModel): # Properties to receive on item modifier creation class ItemModifierCreate(_BaseItemModifier): - createdHoursSinceLaunch: int + pass # Properties to receive on update @@ -23,7 +23,7 @@ class ItemModifierUpdate(_BaseItemModifier): # Properties shared by models stored in DB class ItemModifierInDBBase(_BaseItemModifier): - createdHoursSinceLaunch: int + pass # Properties to return to client diff --git a/src/backend_api/app/crud/__init__.py b/src/backend_api/app/crud/__init__.py index e0b656aa..168976af 100644 --- a/src/backend_api/app/crud/__init__.py +++ b/src/backend_api/app/crud/__init__.py @@ -1,5 +1,12 @@ +from backend_api.app.core.schemas.item_availability import ( + ItemAvailability, + ItemAvailabilityCreate, + ItemAvailabilityUpdate, +) + from app.core.models.models import Currency as model_Currency from app.core.models.models import Item as model_Item +from app.core.models.models import ItemAvailability as model_ItemAvailability from app.core.models.models import ItemBaseType as model_ItemBaseType from app.core.models.models import ItemModifier as model_ItemModifier from app.core.models.models import League as model_League @@ -42,6 +49,16 @@ create_schema=CurrencyCreate, ) +CRUD_itemAvailability = CRUDBase[ + model_ItemAvailability, + ItemAvailability, + ItemAvailabilityCreate, + ItemAvailabilityUpdate, +]( + model=model_ItemAvailability, + schema=ItemAvailability, + create_schema=ItemAvailabilityCreate, +) CRUD_itemBaseType = CRUDBase[ model_ItemBaseType, ItemBaseType, ItemBaseTypeCreate, ItemBaseTypeUpdate From a023e3e72f3dc4005cc23552a65b0aab2601028e Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Thu, 17 Sep 2026 10:26:57 -0230 Subject: [PATCH 03/16] #830 First attempt at new cache based filtering, needs unit tests --- .../external_data_retrieval/config.py | 2 + .../data_retrieval/poe_api_handler.py | 99 +---- .../data_retrieval/schemas/__init__.py | 0 .../data_retrieval/schemas/poe_schema.py | 247 +++++++++++++ .../external_data_retrieval/detectors/base.py | 163 ++------- .../detectors/detector_controller.py | 346 ++++++++++++++++++ .../detectors/unique_detector.py | 57 ++- 7 files changed, 665 insertions(+), 249 deletions(-) create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/__init__.py create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/poe_schema.py create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py index 10bf8d42..72c8dd42 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py @@ -12,6 +12,8 @@ class Settings(BaseSettings): env_file=".env", env_ignore_empty=True, extra="ignore" ) + TAG: str + DOMAIN: str @computed_field # type: ignore[prop-decorator] diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py index 87435c24..cbc68433 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py @@ -16,9 +16,11 @@ ByteResponse, RateLimiterThreadSafe, ) +from data_retrieval_app.external_data_retrieval.detectors.detector_controller import ( + DetectorController, +) from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( UniqueArmourDetector, - UniqueDetector, UniqueJewelDetector, UniqueJewelleryDetector, UniqueUnidentifiedDetector, @@ -40,7 +42,7 @@ class PoEAPIHandler: """ headers = { - "User-Agent": f"OAuth pathofmodifiers/0.1.0 (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" + "User-Agent": f"OAuth pathofmodifiers/{settings.TAG} (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" } def __init__( @@ -49,7 +51,7 @@ def __init__( auth_token: str, *, leagues: list[dict[str, Any]], - item_detectors: list[UniqueDetector] | None = None, + detector_controller: DetectorController | None = None, ) -> None: """ Parameters: @@ -59,15 +61,17 @@ def __init__( """ logger.debug("Initializing PoEAPIHandler.") self.leagues = leagues - if item_detectors is None: - item_detectors = [ - UniqueArmourDetector(leagues), - UniqueJewelDetector(leagues), - UniqueJewelleryDetector(leagues), - UniqueWeaponDetector(leagues), - UniqueUnidentifiedDetector(leagues), - ] - logger.debug("Item detectors set to: " + str(item_detectors)) + if detector_controller is None: + detector_controller = DetectorController( + [ + UniqueArmourDetector(), + UniqueJewelDetector(), + UniqueJewelleryDetector(), + UniqueWeaponDetector(), + UniqueUnidentifiedDetector(), + ], + leagues, + ) self.url = url logger.debug("Url set to: " + self.url) self.auth_token = auth_token @@ -75,75 +79,11 @@ def __init__( logger.debug("Headers set to: " + str(self.headers)) - self.item_detectors = item_detectors - logger.debug("Item detectors set to: " + str(self.item_detectors)) + self.detector_controller = detector_controller self.skip_program_too_slow = False logger.info("PoEAPIHandler successfully initialized.") - def _json_to_df(self, stashes: list) -> pd.DataFrame | None: - df_temp = pd.json_normalize(stashes) - - if "items" not in df_temp.columns: - return None - - df_temp = df_temp.explode(["items"]) - - df_temp = df_temp.loc[~df_temp["items"].isnull()] - - df_temp.drop("items", axis=1, inplace=True) - - df = pd.json_normalize(stashes, record_path=["items"]) - - df["stash_index"] = df_temp.index - - df_temp.index = df.index - - df[df_temp.columns.to_list()] = df_temp - - return df - - def _detector_filter(self, stashes: list) -> pd.DataFrame: - """ - Parameters: - :param stashes: (list) A list of stash objects - """ - df_wanted = pd.DataFrame() - n_new_items = 0 - n_total_unique_items = 0 - df = self._json_to_df(stashes) - if df is None: - return df_wanted - - # The stashes are fed to all item detectors, slowly being filtered down - try: - for item_detector in self.item_detectors: - ( - df_filtered, - item_count, - n_unique_found_items, - df_leftover, - ) = item_detector.iterate_stashes(df) - - df_wanted = pd.concat((df_wanted, df_filtered)) - - del df_filtered - - n_new_items += item_count - n_total_unique_items += n_unique_found_items - if df_leftover.empty: - break - - df = df_leftover.copy(deep=True) - del df_leftover - except Exception as e: - logger.exception( - f"While checking stashes (detector: {item_detector}), this exception occured: {e}" - ) - raise - - return df_wanted.reset_index() - def _get_latest_change_id(self) -> str: """ Gets the latest change id from GGG. @@ -361,9 +301,10 @@ def _read_stream(self) -> tuple[list[Any], str | None]: @sync_timing_tracker def _process_stream(self, stashes: list) -> pd.DataFrame: logger.info("Stashes are ready for processing") - wanted_df = self._detector_filter(stashes) + self.detector_controller._filter_never_used(stashes) + logger.info("Finished processing the data, waiting for more") - return wanted_df + # return wanted_df def _gather_n_checkpoints(self, n: int) -> tuple[pd.DataFrame | None, str | None]: df = None diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/__init__.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/poe_schema.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/poe_schema.py new file mode 100644 index 00000000..5067633a --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/poe_schema.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class PoeModel(BaseModel): + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + ) + + +class CacheItem(PoeModel): + id: str + note: str + + +class CacheStash(PoeModel): + league: str + items: list[CacheItem] + + +class ItemProperty(PoeModel): + name: str + values: list[list[Any]] + display_mode: int | None = Field(default=None, alias="displayMode") + progress: float | None = None + type: int | None = None + suffix: str | None = None + icon: str | None = None + + +class ItemModFlags(PoeModel): + fractured: bool | None = None + mutated: bool | None = None + crafted: bool | None = None + desecrated: bool | None = None + vestigial: bool | None = None + + +class ItemMod(PoeModel): + description: str + flags: ItemModFlags | None = None + + +class Extended(PoeModel): + prefixes: int | None = None + suffixes: int | None = None + + +class Item(PoeModel): + # Basic identity / display + id: str | None = None + name: str + type_line: str = Field(alias="typeLine") + base_type: str = Field(alias="baseType") + rarity: str | None = None + + # Item state + verified: bool + identified: bool + corrupted: bool | None = None + duplicated: bool | None = None + split: bool | None = None + fractured: bool | None = None + synthesised: bool | None = None + elder: bool | None = None + shaper: bool | None = None + searing: bool | None = None + tangled: bool | None = None + abyss_jewel: bool | None = Field( + default=None, + alias="abyssJewel", + ) + delve: bool | None = None + is_relic: bool | None = Field(default=None, alias="isRelic") + replica: bool | None = None + ruthless: bool | None = None + + # Level / league + league: str | None = None + ilvl: int + item_level: int | None = Field(default=None, alias="itemLevel") + monster_level: int | None = Field(default=None, alias="monsterLevel") + + # Inventory dimensions / location + w: int + h: int + x: int | None = None + y: int | None = None + inventory_id: str | None = Field( + default=None, + alias="inventoryId", + ) + + # Stack information + stack_size: int | None = Field(default=None, alias="stackSize") + max_stack_size: int | None = Field( + default=None, + alias="maxStackSize", + ) + stack_size_text: str | None = Field( + default=None, + alias="stackSizeText", + ) + + # Visuals + icon: str + art_filename: str | None = Field( + default=None, + alias="artFilename", + ) + frame_type: int | None = Field( + default=None, + alias="frameType", + ) + frame_type_id: str | None = Field( + default=None, + alias="frameTypeId", + ) + + # Mods + implicit_mods: list[ItemMod] | None = Field( + default=None, + alias="implicitMods", + ) + explicit_mods: list[ItemMod] | None = Field( + default=None, + alias="explicitMods", + ) + crafted_mods: list[str] | None = Field( + default=None, + alias="craftedMods", + ) + fractured_mods: list[str] | None = Field( + default=None, + alias="fracturedMods", + ) + enchant_mods: list[str] | None = Field( + default=None, + alias="enchantMods", + ) + scourge_mods: list[str] | None = Field( + default=None, + alias="scourgeMods", + ) + crucible_mods: list[str] | None = Field( + default=None, + alias="crucibleMods", + ) + veiled_mods: list[str] | None = Field( + default=None, + alias="veiledMods", + ) + cosmetic_mods: list[str] | None = Field( + default=None, + alias="cosmeticMods", + ) + utility_mods: list[str] | None = Field( + default=None, + alias="utilityMods", + ) + + # Item properties + properties: list[ItemProperty] | None = None + notable_properties: list[ItemProperty] | None = Field( + default=None, + alias="notableProperties", + ) + requirements: list[ItemProperty] | None = None + additional_properties: list[ItemProperty] | None = Field( + default=None, + alias="additionalProperties", + ) + next_level_requirements: list[ItemProperty] | None = Field( + default=None, + alias="nextLevelRequirements", + ) + + # Categories / misc + category: dict[str, list[str]] | None = None + flavour_text: list[str] | None = Field( + default=None, + alias="flavourText", + ) + descr_text: str | None = Field( + default=None, + alias="descrText", + ) + sec_descr_text: str | None = Field( + default=None, + alias="secDescrText", + ) + note: str | None = None + forum_note: str | None = None + + # Special item data + extended: Extended | None = None + + # Other flags + locked_to_character: bool | None = Field( + default=None, + alias="lockedToCharacter", + ) + locked_to_account: bool | None = Field( + default=None, + alias="lockedToAccount", + ) + + veiled: bool | None = None + foreseeing: bool | None = None + + def to_cache(self) -> CacheItem: + return CacheItem(id=self.id, note=self.note) + + def __eq__(self, other: CacheItem | object) -> bool: + if isinstance(other, CacheItem): + print(other) + return other.id == self.id and other.note == self.note + return super().__eq__(other) + + +class Stash(PoeModel): + id: str + + public: bool + + stash: str | None = None + stash_type: str = Field(alias="stashType") + league: str | None = None + + items: list[Item] = Field(default_factory=list) + + def to_cache(self) -> CacheStash: + return CacheStash( + league=self.league, items=[item.to_cache() for item in self.items] + ) + + def get_item(self, id: str) -> tuple[int, Item | None]: + for i, item in enumerate(self.items): + if item.id == id: + return i, item + + # this will crash the program if the idx is used + return len(self.items), None diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py index df4bb964..c0ec5acf 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py @@ -1,151 +1,36 @@ -import inspect -import time -from typing import Any +from abc import ABC, abstractmethod -import pandas as pd +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.poe_schema import ( + Item, + Stash, +) -from data_retrieval_app.logs.logger import data_retrieval_logger as logger - -class DetectorBase: +class DetectorBase(ABC): """ Base class for searching stashes for items we want to store and process further """ - wanted_items = {} - found_items = {} - - def __init__( - self, leagues: list[dict[str, Any]], enable_pbar: bool = False - ) -> None: - """ - `self.n_unique_items_found` needs to be stored inbetween item detector sessions. - """ - self.n_unique_items_found = 0 - self.leagues = [league["name"] for league in leagues] + should_cache: bool + wanted_items: list[str] - self.prev_item_hashes_found = {} + @abstractmethod + def _is_wanted(self, item: Item) -> bool: + """Returns true or false based on the criteria of the detector""" - self.pbar_enabled = enable_pbar + def find_interesting_items( + self, donor_stash: Stash, receiver_stash: Stash, no_cache_receiver_stash: Stash + ): + donated_items = list[str]() + for i, item in enumerate(donor_stash.items): + if not self._is_wanted(item): + continue - def _general_filter(self, df: pd.DataFrame) -> pd.DataFrame: - """ - Filtering away items that are never useful - """ - columns = df.columns - if "ruthless" in columns: - df = df.loc[df["ruthless"].isnull()] - if "lockedToCharacter" in columns: - df = df.loc[df["lockedToCharacter"].isnull()] - if "lockedToAccount" in columns: - df = df.loc[df["lockedToAccount"].isnull()] - if "logbookMods" in columns: - df = df.loc[df["logbookMods"].isnull()] - if "crucible" in columns: - df = df.loc[df["crucible"].isnull()] - if "scourged" in columns: - df = df.loc[df["scourged"].isnull()] - if "hybrid" in columns: - df = df.loc[df["hybrid"].isnull()] - if "ultimatumMods" in columns: - df = df.loc[df["ultimatumMods"].isnull()] - if "stash" in columns or "note" in columns: - if "stash" in columns and "note" in columns: - df = df.loc[ - df["stash"].str.startswith(("~b/o", "~price")) - | df["note"].str.startswith(("~b/o", "~price")) - ] - elif "stash" in columns: - df = df.loc[df["stash"].str.startswith(("~b/o", "~price"))] + if self.should_cache: + receiver_stash.items.append(item) else: - df = df.loc[df["note"].str.startswith(("~b/o", "~price"))] - else: - return pd.DataFrame(columns=df.columns) - - df = df.loc[df["league"].isin(self.leagues)] - - return df - - def _specialized_filter(self, df: pd.DataFrame) -> pd.DataFrame: - raise NotImplementedError( - "This method is implemented in child classes. Do not use the base on its own." - ) - - def _filter_on_game_item_id(self, df: pd.DataFrame) -> pd.DataFrame: - """ - The same item can be picked up by the api, without any changes being made to the listing/item, - because the user listed/changed another item in the same tab. We do not want to store these, - as they pain the wrong picture of the market. - - To make sure that only updated items are kept, we also add the note, which contains the price. - Please note that this does not consider relisting of entire tab. - - However, as of right now, these identifiers reset every ~hour. In the future we may pull - the most recent listings from the database instead of resetting. - """ - n_items_before_filter = len(df) - - if "note" in df.columns: - df = df.drop_duplicates(["id", "note"]) - note = df["note"].apply(hash) - else: - df = df.drop_duplicates("id") - note = 0 - - game_item_id = df["id"].apply(int, base=16) - - hashes: pd.Series[int] = game_item_id + note - - unique_hashes = set(hashes) - - if not self.prev_item_hashes_found: - self.prev_item_hashes_found = unique_hashes - else: - duplicate_hashes = self.prev_item_hashes_found.intersection(unique_hashes) - self.prev_item_hashes_found |= unique_hashes - - items_to_drop_mask = ~hashes.isin(duplicate_hashes) - df = df.loc[items_to_drop_mask] - - n_items_filtered = n_items_before_filter - len(df) - - logger.info( - f'detector="{self}" {n_items_before_filter=} {n_items_filtered=} percent_of_total={1 - n_items_filtered / max(1, n_items_before_filter):.2%}' - ) - - return df - - def _snapshot(self, df: pd.DataFrame, filepath: str = None) -> None: - """ - A method for creating a snapshot for a given dataframe. If no filepath is given, it saves the class - this directory with the name `snapshot_{name_of_class}_{time}.csv` - """ - if filepath is None: - filepath = ( - inspect.getfile(DetectorBase)[:-7] - + f"snapshot_{self}_{time.time():.0f}.csv" - ) - - logger.info(f"Saving a snapshot, for the detector {self}") - df.to_csv(filepath, encoding="utf-8") - - def iterate_stashes( - self, df: pd.DataFrame - ) -> tuple[pd.DataFrame, int, int, pd.DataFrame]: - """ - TODO - """ - - df = self._general_filter(df) - if df.empty: - return df, 0, self.n_unique_items_found, df - df_filtered = self._specialized_filter(df) - df_leftover = df.loc[~df.index.isin(df_filtered.index)] - - df_filtered = self._filter_on_game_item_id(df_filtered) - - item_count = len(df_filtered) - if self.pbar_enabled: - self.n_unique_items_found = len(self.found_items.keys()) + no_cache_receiver_stash.items.append(item) + donated_items.append(i) - return df_filtered, item_count, self.n_unique_items_found, df_leftover + for idx in reversed(donated_items): + donor_stash.items.pop(idx) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py new file mode 100644 index 00000000..c5a37eed --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py @@ -0,0 +1,346 @@ +from typing import Any + +import redis +from pydantic import TypeAdapter + +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.poe_schema import ( + CacheItem, + CacheStash, + Item, + Stash, +) +from data_retrieval_app.external_data_retrieval.detectors.base import DetectorBase + + +class DetectorController: + def __init__( + self, + detectors: list[DetectorBase], + leagues: list[dict[str, Any]], + redis_cache: redis.Redis, + ): + self.leagues = [league["name"] for league in leagues] + + self.detectors = detectors + + self.local_cache = dict[str, CacheStash]() + self.local_cache[ + "158c93260022f9b39f9e7c6980711ab057c6ef2e28e1cdf00f2c2b5aef2c880c" + ] = CacheStash( + league="Standard", + items=[ + CacheItem( # changed price + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_updated", + note="~b/o 2 mirror", + ), + CacheItem( # same price + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402", + note="~b/o 30 chaos", + ), + CacheItem( # removed + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_removed", + note="~b/o 2 mirror", + ), + ], + ) + self.cached_stash_adapter = TypeAdapter(list[CacheStash | None]) + self.redis_cache = redis_cache + + def _filter_duplicates(self, stashes: list[Stash]) -> list[Stash]: + previous_ids = list[str]() + non_duplicate_stashes = list[Stash]() + for stash in stashes: + if stash.id in previous_ids: + continue + + previous_ids.append(stash.id) + non_duplicate_stashes.append(stash) + + return non_duplicate_stashes + + def _filter_never_used( + self, stashes: list[Stash] + ) -> tuple[list[Stash], list[Stash]]: + uninteresting_item_fields = { + "monsterLevel", + "lockedToCharacter", + "lockedToAccount", + "logbookMods", + "ultimatumMods", + "mercenarySkills", + "isRelic", + "enshrouded", + "ruthless", + "hybrid", + "unmodifiableExceptChaos", + } + + def has_price(note: str): + return note is not None and note.startswith(("~b/o", "~price")) + + filtered_stashes = list[Stash]() + empty_stashes = list[Stash]() + for stash in stashes: + items: list[Item] = stash.items + filtered_items = [] + if not stash.public or stash.league not in self.leagues: + # TODO should I already filter leagues? depends on what happens on migration + print(stash.id) + continue + for item in items: + if ( + uninteresting_item_fields + & item.model_dump(exclude_none=True).keys() + ): + continue + + if has_price(item.note): + pass + elif has_price(stash.stash): + item.note = stash.stash + else: + continue + + filtered_items.append(item) + + # print(len(filtered_items), stash.id) + if filtered_items: + stash.items = filtered_items + filtered_stashes.append(stash) + else: + stash.items = [] + empty_stashes.append(stash) + + return filtered_stashes, empty_stashes + + def _filter_uninteresting_items( + self, stashes: list[Stash] + ) -> tuple[list[Stash], list[Item], list[Stash]]: + interesting_stashes = list[Stash]() + no_cache_new_items = list[Stash]() + filtered_stashes = list[Stash]() + for donor_stash in stashes: + receiver_stash = donor_stash.model_copy(update={"items": []}) + no_cache_receiver_stash = donor_stash.model_copy(update={"items": []}) + for detector in self.detectors: + # the donor stash is modified inplace, with its items moving over to the receiver stashes inplace + detector.find_interesting_items( + donor_stash, receiver_stash, no_cache_receiver_stash + ) + + if receiver_stash.items: + interesting_stashes.append(receiver_stash) + elif no_cache_receiver_stash.items: + # items such as unidentified items + no_cache_new_items.extend(no_cache_receiver_stash.items) + else: + filtered_stashes.append(donor_stash) + + return interesting_stashes, no_cache_new_items, filtered_stashes + + def _get_cached_stashes(self, ids: list[str]) -> list[CacheStash | None]: + """ + First checks local cache, then redis cache. This method relies on the local cache + being exactly aligned with the redis cache for stashes that exists in both. + """ + if len(ids) == 0: + return [] + + # first local cache + cached_stashes = [] + still_needs_check = list[tuple[int, str]]() + for i, id in enumerate(ids): + cached_stash = self.local_cache.get(id) + if cached_stash is None: + still_needs_check.append((i, id)) + cached_stashes.append(cached_stash) + + # then redis cache + redis_cached_stashes = self.cached_stash_adapter.validate_python( + self.redis_cache.mget([id for _, id in still_needs_check]) + ) + for i, cached_stash in enumerate(redis_cached_stashes): + if cached_stash is not None: + idx, _ = still_needs_check[i] + cached_stashes[idx] = cached_stash + + return cached_stashes + + def _find_removed_items(self, empty_stash_ids: list[str]) -> list[CacheItem]: + # TODO what if league can change but id stays the same + empty_cached_stashes = self._get_cached_stashes(empty_stash_ids) + removed_items = list[CacheItem]() + cached_stashes_to_remove = list[str]() + for id, cached_stash in zip(empty_stash_ids, empty_cached_stashes, strict=True): + if cached_stash is not None: + print("cached stash must be removed", id) + cached_stashes_to_remove.append(id) + removed_items.extend(cached_stash.items) + + [self.local_cache.pop(id, None) for id in cached_stashes_to_remove] + self.redis_cache.delete(cached_stashes_to_remove) + + return removed_items + + def _find_changed_items_filter_unchanged( + self, stashes: list[Stash] + ) -> tuple[list[Item], list[Item], list[CacheItem]]: + # TODO what if league can change but id stays the same + not_empty_stash_ids = [stash.id for stash in stashes] + + not_empty_cached_stashes = self._get_cached_stashes(not_empty_stash_ids) + + changed_items = list[Item]() + new_items = list[Item]() + removed_items = list[CacheItem]() + + stashes_to_cache = dict[str, CacheStash]() + for cached_stash, stash in zip(not_empty_cached_stashes, stashes, strict=True): + cached_stash_changed = False + if cached_stash is not None: + for cached_item in cached_stash.items[:]: + idx, new_item = stash.get_item(cached_item.id) + if new_item is None: + # a cached item doesnt exist anymore + print("cached item no longer exists", cached_item.id) + cached_stash_changed = True + cached_stash.items.remove(cached_item) + removed_items.append(cached_item) + continue + if new_item.note == cached_item.note: + # Duplicate found in cache + print("duplicate found", new_item.id) + stash.items.pop(idx) + continue + + # Item has changed from cache + print("item has changed", new_item.id) + changed_items.append(new_item) + cached_stash_changed = True + cached_stash.items.remove(cached_item) + + cached_stash.items.extend(stash.to_cache().items) + + else: + # The stash has never been cached + cached_stash_changed = True + cached_stash = stash.to_cache() + + new_items.extend(stash.items) + + if cached_stash_changed and cached_stash.items: + stashes_to_cache[stash.id] = cached_stash + + self.local_cache.update(stashes_to_cache) + self.redis_cache.mset(stashes_to_cache) + + return changed_items, new_items, removed_items + + def filter_stashes( + self, stashes: list[Stash] + ) -> tuple[list[CacheItem], list[Item], list[Item]]: + """ + Returns: + Removed items: items which need to be closed + Changed items: items which need to start a new entry + New items: items which need to be entered into the system + """ + stashes = self._filter_duplicates(stashes) + + filtered_stashes, empty_stashes = self._filter_never_used(stashes) + empty_stash_ids = [stash.id for stash in empty_stashes] + + ( + interesting_stashes, + new_items, + uninteresting_stashes, + ) = self._filter_uninteresting_items(filtered_stashes) + empty_stash_ids.extend([stash.id for stash in uninteresting_stashes]) + + removed_items = self._find_removed_items(empty_stash_ids) + + ( + changed_items, + more_new_items, + more_removed_items, + ) = self._find_changed_items_filter_unchanged(interesting_stashes) + new_items.extend(more_new_items) + removed_items.extend(more_removed_items) + + return removed_items, changed_items, new_items + + +if __name__ == "__main__": + import json + + from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( + UniqueArmourDetector, + UniqueJewelDetector, + UniqueJewelleryDetector, + UniqueUnidentifiedDetector, + UniqueWeaponDetector, + ) + + class RedisCacheDecoy: + cache = { + "3ca7f08b3382795478397c41d110fc407da007bffd7e88dd9e6501bb306c7a6a": { + # must be removed + "league": "Standard", + "items": [ + { # doesnt exist anymore + "id": "1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402", + "note": "~b/o 30 chaos", + } + ], + }, + "eca1a8b071f18a3f07aa55f914173287a445f46b8e433ed3e04edf636dc6b946": { + "league": "Allflame", + "items": [ + { # duplicate + "id": "5588417cffcfbd0efe651cde0e5c606867d2b58a34812b90ce78e1352bacf68a", + "note": "~b/o 32 divine", + } + ], + }, + } + + def mget(self, keys: list[str]): + return [self.cache.get(key) for key in keys] + + def mset(self, key_to_object: dict[str, Any]): + for key, obj in key_to_object.items(): + self.cache[key] = obj + + def delete(self, keys: list[str]): + for key in keys: + self.cache.pop(key, None) + + detector_controller = DetectorController( + [ + UniqueArmourDetector(), + UniqueJewelDetector(), + UniqueJewelleryDetector(), + UniqueWeaponDetector(), + UniqueUnidentifiedDetector(), + ], + [{"name": "Standard"}, {"name": "Hardcore"}, {"name": "Allflame"}], + redis_cache=RedisCacheDecoy(), + ) + + stashes = json.load(open("stashes.json")) + stashes = [ + stash + for stash in stashes + if stash["id"] + in ( + "b3e32cdca2367d795b6d37b625bf22096ea2ee405b20357dc80c406dd0e4b63e", # empty + "ea0a95a395b48875e81f947cc87e315ba4a719fb08a5453f70c9e8c6fd2d0266", # uninteresting + "a68ead45e8bf786cec6c52d37a226739b5797f1c0264cf64321635f448dac97e", # interesting + "158c93260022f9b39f9e7c6980711ab057c6ef2e28e1cdf00f2c2b5aef2c880c", # interesting + "3ca7f08b3382795478397c41d110fc407da007bffd7e88dd9e6501bb306c7a6a", # uninteresting + "eca1a8b071f18a3f07aa55f914173287a445f46b8e433ed3e04edf636dc6b946", # interesting + ) + ] + stash_adapter = TypeAdapter(list[Stash]) + stashes = stash_adapter.validate_python(stashes) + detector_controller.filter_stashes(stashes) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py index 11820354..d2c6eab5 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py @@ -1,28 +1,19 @@ -import pandas as pd - +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.poe_schema import ( + Item, +) from data_retrieval_app.external_data_retrieval.detectors.base import DetectorBase class UniqueDetector(DetectorBase): - def _check_if_wanted(self, df: pd.DataFrame) -> pd.DataFrame: - df = df.loc[df["name"].isin(self.wanted_items)] - return df - - def _specialized_filter(self, df: pd.DataFrame) -> pd.DataFrame: - if "rarity" not in df.columns: - return pd.DataFrame(columns=df.columns) - - df = df.loc[df["rarity"] == "Unique"] - - df = self._check_if_wanted(df) + should_cache = True - if self.pbar_enabled: - temp_df = df["name"] + df["baseType"] - for name_baseType in temp_df.unique(): - if name_baseType not in self.found_items: - self.found_items[name_baseType] = True + def _is_wanted(self, item: Item) -> bool: + if item.rarity is None or item.rarity != "Unique": + return False + if item.name not in self.wanted_items: + return False - return df + return True class UniqueUnidentifiedDetector(UniqueDetector): @@ -40,6 +31,8 @@ class UniqueUnidentifiedDetector(UniqueDetector): No, because they are not particularly sought after in their un-id form """ + should_cache = False + wanted_base_types = [ "Viridian Jewel", "Cobalt Jewel", @@ -96,20 +89,22 @@ class UniqueUnidentifiedDetector(UniqueDetector): "Soulcord.png": "Screams of the Desiccated", } - def _check_if_wanted(self, df: pd.DataFrame) -> pd.DataFrame: + def _is_wanted(self, item: Item) -> bool: """ - Uses the icon to identify which unique it is, then saving that name. - If the name attribute still has a length of 0 it means no matching unique - was found. + Uses the icon to identify which unique it is, then saving that name inplace. """ - df = df.loc[(~df["identified"] & df["baseType"].isin(self.wanted_base_types))] - - for icon, name in self.wanted_item_icons.items(): - df.loc[df["icon"].str.endswith(icon), "name"] = name - - df = df.loc[df["name"].str.len() != 0] - # self._snapshot(df) - return df + if item.identified: + return False + if item.base_type not in self.wanted_base_types: + return False + + icon = item.icon.split("/")[-1] + name = self.wanted_item_icons.get(icon) + if name is None: + return False + + item.name = name + return True def __str__(self): return "Unidentifed Unique detector" From 4ed3b36e3dbd414c20f6a97028f3d214e578fedb Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 19 Sep 2026 12:36:25 -0230 Subject: [PATCH 04/16] #830 Updated the currency database representation, now divides it into types and prices --- .../dd4abb2b6541_optimize_item_table.py | 17 +- ...it_currency_table_into_prices_and_types.py | 130 +++++++++++++ src/backend_api/app/api/routes/currency.py | 137 ++++++-------- src/backend_api/app/core/models/models.py | 36 ++-- src/backend_api/app/core/schemas/__init__.py | 12 +- src/backend_api/app/core/schemas/currency.py | 63 ++++++- src/backend_api/app/crud/__init__.py | 45 +---- .../app/crud/extensions/crud_currency.py | 160 ++++++++++------ .../app/crud/extensions/crud_league.py | 3 + .../app/crud/extensions/crud_modifier.py | 7 + .../crud/extensions/crud_unidentifiedItem.py | 7 + src/backend_api/app/plotting/plotter.py | 23 ++- src/backend_data_retrieval/Dockerfile | 7 + .../currency/currency_type_data_depositor.py | 176 ++++++++++++++++++ .../data_deposit/data_depositor_base.py | 5 +- .../data_retrieval_app/data_deposit/main.py | 4 + 16 files changed, 625 insertions(+), 207 deletions(-) create mode 100644 src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py create mode 100644 src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py diff --git a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py index f8bdbee4..df3df289 100644 --- a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py +++ b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py @@ -1,7 +1,7 @@ """optimize item table Revision ID: dd4abb2b6541 -Revises: 965e766db0a0 +Revises: ee1239y6yfda Create Date: 2026-07-23 15:20:18.565656 """ @@ -13,7 +13,7 @@ # revision identifiers, used by Alembic. revision: str = "dd4abb2b6541" -down_revision: Union[str, None] = "965e766db0a0" +down_revision: Union[str, None] = "ee1239y6yfda" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -26,14 +26,14 @@ def upgrade() -> None: "temp_league_id", sa.SmallInteger(), nullable=False ), # Temporary column # sa.Column("itemId", sa.Integer(), nullable=False), # Will be added later - sa.Column("currencyId", sa.Integer(), nullable=False), + sa.Column("currencyId", sa.SmallInteger(), nullable=False), sa.Column("currencyAmount", sa.Float(4), nullable=False), sa.Column("isAsync", sa.Boolean()), sa.Column("validFrom", sa.SmallInteger(), nullable=False), sa.Column("validTo", sa.SmallInteger()), sa.ForeignKeyConstraint( ["currencyId"], - ["currency.currencyId"], + ["currency_type.currencyId"], ondelete="RESTRICT", onupdate="CASCADE", ), @@ -216,14 +216,14 @@ def upgrade() -> None: primary_key=True, ), sa.Column("itemId", sa.Integer(), nullable=False), - sa.Column("currencyId", sa.Integer(), nullable=False), + sa.Column("currencyId", sa.SmallInteger(), nullable=False), sa.Column("currencyAmount", sa.Float(4), nullable=False), sa.Column("isAsync", sa.Boolean()), sa.Column("validFrom", sa.SmallInteger(), nullable=False), sa.Column("validTo", sa.SmallInteger()), sa.ForeignKeyConstraint( ["currencyId"], - ["currency.currencyId"], + ["currency_type.currencyId"], ondelete="RESTRICT", onupdate="CASCADE", ), @@ -301,6 +301,11 @@ def upgrade() -> None: ondelete="RESTRICT", onupdate="CASCADE", ) + batch_op.create_index( + "ix_item_modifier_modifierId_itemId", + ["modifierId", "itemId"], + unique=False, + ) def downgrade() -> None: diff --git a/src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py b/src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py new file mode 100644 index 00000000..3d53b103 --- /dev/null +++ b/src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py @@ -0,0 +1,130 @@ +"""optimize item table + +Revision ID: ee1239y6yfda +Revises: 965e766db0a0 +Create Date: 2026-09-18 19:54:18.565656 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "ee1239y6yfda" +down_revision: Union[str, None] = "965e766db0a0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "currency_type", + sa.Column( + "currencyId", + sa.SmallInteger(), + sa.Identity(always=False, start=1, increment=1), + nullable=False, + primary_key=True, + ), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("tradeName", sa.Text(), nullable=False), + ) + + op.execute(""" + INSERT INTO currency_type (name, "tradeName") + SELECT DISTINCT 'TEMP_NAME', c."tradeName" + FROM currency c; + """) + + op.create_table( + "currency_price", + sa.Column( + "currencyId", + sa.SmallInteger(), + nullable=False, + ), + sa.Column("leagueId", sa.SmallInteger(), nullable=False), + sa.Column("createdHoursSinceLaunch", sa.SmallInteger(), nullable=False), + sa.Column("valueInChaos", sa.Float(4), nullable=False), + sa.PrimaryKeyConstraint("currencyId", "leagueId", "createdHoursSinceLaunch"), + sa.ForeignKeyConstraint( + ["currencyId"], + ["currency_type.currencyId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ), + sa.ForeignKeyConstraint( + ["leagueId"], ["league.leagueId"], ondelete="CASCADE", onupdate="CASCADE" + ), + ) + + op.execute(""" + INSERT INTO currency_price + SELECT + ct."currencyId", + c."leagueId", + c."createdHoursSinceLaunch", + c."valueInChaos" + FROM currency c + JOIN currency_type ct + ON c."tradeName" = ct."tradeName"; + """) + + op.drop_constraint("item_currencyId_fkey", "item", type_="foreignkey") + op.drop_constraint("item_currencyId_fkey1", "item", type_="foreignkey") + op.execute(""" + UPDATE item i + SET "currencyId"=ct."currencyId" + FROM currency c + JOIN currency_type ct + ON ct."tradeName" = c."tradeName" + WHERE i."currencyId" = c."currencyId"; + """) + + op.drop_constraint( + "unidentified_item_currencyId_fkey", "unidentified_item", type_="foreignkey" + ) + op.execute(""" + UPDATE unidentified_item ui + SET "currencyId"=ct."currencyId" + FROM currency c + JOIN currency_type ct + ON ct."tradeName" = c."tradeName" + WHERE ui."currencyId" = c."currencyId"; + """) + + op.drop_table("currency") + + op.alter_column( + "item", + "currencyId", + existing_type=sa.Integer(), + type_=sa.SmallInteger(), + ) + op.create_foreign_key( + None, + "item", + "currency_type", + ["currencyId"], + ["currencyId"], + ) + + op.alter_column( + "unidentified_item", + "currencyId", + existing_type=sa.Integer(), + type_=sa.SmallInteger(), + ) + op.create_foreign_key( + None, + "unidentified_item", + "currency_type", + ["currencyId"], + ["currencyId"], + ) + + +def downgrade() -> None: + raise NotImplementedError("Downgrade is not supported for this migration") diff --git a/src/backend_api/app/api/routes/currency.py b/src/backend_api/app/api/routes/currency.py index d9e3ed03..6ee0e962 100644 --- a/src/backend_api/app/api/routes/currency.py +++ b/src/backend_api/app/api/routes/currency.py @@ -4,16 +4,12 @@ from sqlalchemy.orm import Session import app.core.schemas as schemas -from app.api.api_message_util import ( - get_delete_return_msg, -) from app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) from app.api.params import FilterParams -from app.core.models.models import Currency from app.core.rate_limit.rate_limit_config import rate_limit_settings from app.core.rate_limit.rate_limiters import ( apply_user_rate_limits, @@ -28,44 +24,35 @@ @router.get( - "/{currencyId}", - response_model=schemas.Currency, + "/price/", + response_model=schemas.CurrencyPrice | list[schemas.CurrencyPrice], dependencies=[ Depends(get_current_active_user), ], ) -@apply_user_rate_limits( - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_MINUTE, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_HOUR, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_DAY, -) -async def get_currency( - request: Request, # noqa: ARG001 - response: Response, # noqa: ARG001 - currencyId: int, +async def get_all_currency_prices( + filter_params: Annotated[FilterParams, Query()], db: Session = Depends(get_db), ): """ - Get currency by key and value for "currencyId". + Get all currencies. - Always returns one currency. + Returns a list of all currencies. """ - currency_map = {"currencyId": currencyId} - currency = await CRUD_currency.get(db=db, filter=currency_map) + all_currencies = await CRUD_currency.get_prices(db=db, filter_params=filter_params) - return currency + return all_currencies @router.get( - "/", - response_model=schemas.Currency | list[schemas.Currency], + "/type/", + response_model=schemas.CurrencyType | list[schemas.CurrencyType], dependencies=[ - Depends(get_current_active_superuser), + Depends(get_current_active_user), ], ) -async def get_all_currencies( +async def get_all_currency_types( filter_params: Annotated[FilterParams, Query()], db: Session = Depends(get_db), ): @@ -75,36 +62,33 @@ async def get_all_currencies( Returns a list of all currencies. """ - all_currencies = await CRUD_currency.get(db=db, filter_params=filter_params) + all_currencies = await CRUD_currency.get_types(db=db, filter_params=filter_params) return all_currencies @router.get( - "/latest_currency_id/", - response_model=int, - tags=["latest_currency_id"], + "/", + response_model=list[schemas.Currency], dependencies=[ Depends(get_current_active_user), ], ) -@apply_user_rate_limits( - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_MINUTE, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_HOUR, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_DAY, -) -async def get_latest_currency_id( - request: Request, # noqa: ARG001 - response: Response, # noqa: ARG001 +async def get_all_currency( + filter_params: Annotated[FilterParams, Query()], db: Session = Depends(get_db), ): """ - Get the latest currencyId, returns 1 if table is empty + Get all currencies. - Can only be used safely on an empty table or directly after an insertion. + Returns a list of all currencies. """ - return await CRUD_currency.get_latest_currency_id(db) + + all_currencies = await CRUD_currency.get_currency_from_query( + db=db, filter_params=filter_params + ) + + return all_currencies @router.get( @@ -136,9 +120,8 @@ async def get_latest_hours( @router.get( - "/latest_currencies/", + "/price/latest/", response_model=list[schemas.Currency], - tags=["latest_currencies"], dependencies=[ Depends(get_current_active_user), ], @@ -187,14 +170,14 @@ async def get_currency_from_query( @router.post( - "/", - response_model=schemas.CurrencyCreate | list[schemas.CurrencyCreate] | None, + "/type/", + response_model=schemas.CurrencyTypeCreate | list[schemas.CurrencyTypeCreate] | None, dependencies=[ Depends(get_current_active_superuser), ], ) -async def create_currency( - currency: schemas.CurrencyCreate | list[schemas.CurrencyCreate], +async def create_currency_type( + currency: schemas.CurrencyTypeCreate | list[schemas.CurrencyTypeCreate], return_nothing: bool | None = None, db: Session = Depends(get_db), ): @@ -204,59 +187,53 @@ async def create_currency( Returns the created currency or list of currencies. """ - return await CRUD_currency.create( + return await CRUD_currency.create_types( db=db, obj_in=currency, return_nothing=return_nothing ) -@router.put( - "/", - response_model=schemas.Currency, +@router.post( + "/price/", + response_model=schemas.CurrencyPriceCreate + | list[schemas.CurrencyPriceCreate] + | None, dependencies=[ Depends(get_current_active_superuser), ], ) -async def update_currency( - currencyId: int, - currency_update: schemas.CurrencyUpdate, +async def create_currency_price( + currency: schemas.CurrencyPriceCreate | list[schemas.CurrencyPriceCreate], + return_nothing: bool | None = None, db: Session = Depends(get_db), ): """ - Update a currency by key and value for "currencyId". + Create one or a list of currencies. - Returns the updated currency. + Returns the created currency or list of currencies. """ - currency_map = {"currencyId": currencyId} - currency = await CRUD_currency.get( - db=db, - filter=currency_map, + return await CRUD_currency.create_prices( + db=db, obj_in=currency, return_nothing=return_nothing ) - return await CRUD_currency.update(db_obj=currency, obj_in=currency_update, db=db) - -@router.delete( - "/", - response_model=str, - dependencies=[ - Depends(get_current_active_superuser), - ], +@router.put( + "/type/", + response_model=schemas.CurrencyType, + dependencies=[Depends(get_current_active_superuser)], ) -async def delete_currency( +async def update_modifier( currencyId: int, + currency_type_update: schemas.CurrencyTypeUpdate, db: Session = Depends(get_db), ): - """ - Delete a currency by key and value for "currencyId". - - Returns a message indicating the currency was deleted. - Always deletes one currency. - """ + currency_type_map = {"currencyId": currencyId} - currency_map = {"currencyId": currencyId} - await CRUD_currency.remove(db=db, filter=currency_map) + currency_type = await CRUD_currency.get_types( + db=db, + filter=currency_type_map, + ) - return get_delete_return_msg( - model_table_name=Currency.__tablename__, filter=currency_map - ).message + return await CRUD_currency.update_type( + db_obj=currency_type, obj_in=currency_type_update, db=db + ) diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index 4644b80c..21f34b91 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -35,18 +35,32 @@ class League(Base): version: Mapped[float] = mapped_column(Float, nullable=False) -class Currency(Base): - __tablename__ = "currency" - - currencyId: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True) - createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) - valueInChaos: Mapped[float] = mapped_column(Float(4), nullable=False) +class CurrencyType(Base): + __tablename__ = "currency_type" + currencyId: Mapped[int] = mapped_column(SmallInteger, Identity(), primary_key=True) + name: Mapped[str] = mapped_column(Text, nullable=False) tradeName: Mapped[str] = mapped_column(Text, nullable=False) - leagueId: Mapped[SmallInteger] = mapped_column( + + +class CurrencyPrice(Base): + __tablename__ = "currency_price" + + currencyId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey("currency_type.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), + nullable=False, + ) + leagueId: Mapped[int] = mapped_column( SmallInteger, ForeignKey("league.leagueId", ondelete="RESTRICT", onupdate="CASCADE"), nullable=False, ) + createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) + valueInChaos: Mapped[float] = mapped_column(Float(4), nullable=False) + + __table_args__ = ( + PrimaryKeyConstraint("currencyId", "leagueId", "createdHoursSinceLaunch"), + ) class ItemBaseType(Base): @@ -81,7 +95,7 @@ class ItemAvailability(Base): currencyId: Mapped[int] = mapped_column( Integer, - ForeignKey("currency.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), + ForeignKey("currency_type.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), nullable=False, ) currencyAmount: Mapped[float] = mapped_column(Float(4), nullable=False) @@ -163,7 +177,7 @@ class _ItemBase: currencyId: Mapped[int] = mapped_column( Integer, - ForeignKey("currency.currencyId", ondelete="RESTRICT"), + ForeignKey("currency_type.currencyId", ondelete="RESTRICT"), index=True, nullable=False, ) @@ -310,10 +324,8 @@ class ItemModifier(Base): onupdate="CASCADE", ), Index( - "ix_item_modifierId_createdHoursSinceLaunch_roll_itemId", + "ix_item_modifierId_itemId", "modifierId", - "createdHoursSinceLaunch", - "roll", "itemId", ), ) diff --git a/src/backend_api/app/core/schemas/__init__.py b/src/backend_api/app/core/schemas/__init__.py index 5df53cdc..83ee7ebf 100644 --- a/src/backend_api/app/core/schemas/__init__.py +++ b/src/backend_api/app/core/schemas/__init__.py @@ -1,6 +1,16 @@ "All schemas are imported here and then exported to the main file" -from .currency import Currency, CurrencyCreate, CurrencyInDB, CurrencyUpdate +from .currency import ( + CurrencyPrice, + CurrencyPriceCreate, + CurrencyPriceInDB, + CurrencyPriceUpdate, + CurrencyType, + CurrencyTypeCreate, + CurrencyTypeInDB, + CurrencyTypeUpdate, + Currency, +) from .item import Item, ItemCreate, ItemInDB, ItemUpdate from .item_availability import ( ItemAvailability, diff --git a/src/backend_api/app/core/schemas/currency.py b/src/backend_api/app/core/schemas/currency.py index 0f09d102..4d9f07ec 100644 --- a/src/backend_api/app/core/schemas/currency.py +++ b/src/backend_api/app/core/schemas/currency.py @@ -2,41 +2,84 @@ # Shared currency props -class _BaseCurrency(_pydantic.BaseModel): +class _BaseCurrencyPrice(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) - tradeName: str - valueInChaos: float + currencyId: int leagueId: int + createdHoursSinceLaunch: int + valueInChaos: float # Properties to receive on currency creation -class CurrencyCreate(_BaseCurrency): - createdHoursSinceLaunch: int +class CurrencyPriceCreate(_BaseCurrencyPrice): + pass # Properties to receive on update -class CurrencyUpdate(_BaseCurrency): +class CurrencyPriceUpdate(_BaseCurrencyPrice): pass # Properties shared by models stored in DB -class CurrencyInDBBase(_BaseCurrency): - createdHoursSinceLaunch: int +class CurrencyPriceInDBBase(_BaseCurrencyPrice): + pass + + +# Properties to return to client +class CurrencyPrice(CurrencyPriceInDBBase): + pass + + +# Properties stored in DB +class CurrencyPriceInDB(CurrencyPriceInDBBase): + pass + + +# Shared currency props +class _BaseCurrencyType(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + + name: str + tradeName: str + + +# Properties to receive on currency creation +class CurrencyTypeCreate(_BaseCurrencyType): + pass + + +# Properties to receive on update +class CurrencyTypeUpdate(_BaseCurrencyType): + currencyId: int + + +# Properties shared by models stored in DB +class CurrencyTypeInDBBase(_BaseCurrencyType): currencyId: int # Properties to return to client -class Currency(CurrencyInDBBase): +class CurrencyType(CurrencyTypeInDBBase): pass # Properties stored in DB -class CurrencyInDB(CurrencyInDBBase): +class CurrencyTypeInDB(CurrencyTypeInDBBase): pass +class Currency(_pydantic.BaseModel): + currencyId: int + tradeName: str + + leagueId: int + createdHoursSinceLaunch: int + valueInChaos: float + + class CurrencyQuery(_pydantic.BaseModel): createdHoursSinceLaunch: int | None = None + currencyId: int | None = None tradeName: str | None = None leagueId: int | None = None diff --git a/src/backend_api/app/crud/__init__.py b/src/backend_api/app/crud/__init__.py index 168976af..3d6e467f 100644 --- a/src/backend_api/app/crud/__init__.py +++ b/src/backend_api/app/crud/__init__.py @@ -1,19 +1,13 @@ -from backend_api.app.core.schemas.item_availability import ( - ItemAvailability, - ItemAvailabilityCreate, - ItemAvailabilityUpdate, -) - -from app.core.models.models import Currency as model_Currency from app.core.models.models import Item as model_Item from app.core.models.models import ItemAvailability as model_ItemAvailability from app.core.models.models import ItemBaseType as model_ItemBaseType from app.core.models.models import ItemModifier as model_ItemModifier -from app.core.models.models import League as model_League -from app.core.models.models import Modifier as model_Modifier -from app.core.models.models import UnidentifiedItem as model_UnidentifiedItem -from app.core.schemas.currency import Currency, CurrencyCreate from app.core.schemas.item import Item, ItemCreate, ItemUpdate +from app.core.schemas.item_availability import ( + ItemAvailability, + ItemAvailabilityCreate, + ItemAvailabilityUpdate, +) from app.core.schemas.item_base_type import ( ItemBaseType, ItemBaseTypeCreate, @@ -24,15 +18,6 @@ ItemModifierCreate, ItemModifierUpdate, ) -from app.core.schemas.league import ( - League, - LeagueCreate, -) -from app.core.schemas.modifier import Modifier, ModifierCreate -from app.core.schemas.unidentified_item import ( - UnidentifiedItem, - UnidentifiedItemCreate, -) from app.crud.extensions.crud_currency import CRUDCurrency from app.crud.extensions.crud_league import CRUDLeague from app.crud.extensions.crud_modifier import CRUDModifier @@ -41,13 +26,9 @@ from .base import CRUDBase -CRUD_league = CRUDLeague(model=model_League, schema=League, create_schema=LeagueCreate) +CRUD_league = CRUDLeague() -CRUD_currency = CRUDCurrency( - model=model_Currency, - schema=Currency, - create_schema=CurrencyCreate, -) +CRUD_currency = CRUDCurrency() CRUD_itemAvailability = CRUDBase[ model_ItemAvailability, @@ -83,16 +64,8 @@ ItemUpdate, ](model=model_Item, schema=Item, create_schema=ItemCreate) -CRUD_unidentifiedItem = CRUDUnidentifiedItem( - model=model_UnidentifiedItem, - schema=UnidentifiedItem, - create_schema=UnidentifiedItemCreate, -) +CRUD_unidentifiedItem = CRUDUnidentifiedItem() -CRUD_modifier = CRUDModifier( - model=model_Modifier, - schema=Modifier, - create_schema=ModifierCreate, -) +CRUD_modifier = CRUDModifier() CRUD_user = CRUDUser() diff --git a/src/backend_api/app/crud/extensions/crud_currency.py b/src/backend_api/app/crud/extensions/crud_currency.py index 3a094bb9..80f7689a 100644 --- a/src/backend_api/app/crud/extensions/crud_currency.py +++ b/src/backend_api/app/crud/extensions/crud_currency.py @@ -1,49 +1,69 @@ from pydantic import TypeAdapter from sqlalchemy import select from sqlalchemy.orm import Session -from sqlalchemy.sql.expression import and_, func, or_ +from sqlalchemy.sql.expression import ColumnElement, and_, func, or_ -from app.core.models.models import Currency as model_Currency +from app.api.params import FilterParams +from app.core.models.models import CurrencyPrice as model_CurrencyPrice +from app.core.models.models import CurrencyType as model_CurrencyType from app.core.schemas.currency import ( Currency, - CurrencyCreate, + CurrencyPrice, + CurrencyPriceCreate, + CurrencyPriceUpdate, CurrencyQuery, - CurrencyUpdate, + CurrencyType, + CurrencyTypeCreate, + CurrencyTypeUpdate, ) from app.crud.base import CRUDBase -class CRUDCurrency( - CRUDBase[ - model_Currency, - Currency, - CurrencyCreate, - CurrencyUpdate, - ] -): - async def get_latest_currency_id(self, db: Session) -> int: - stmt = select( - func.max(model_Currency.currencyId).label("latestCurrencyId") - ).limit(1) +class CRUDCurrency: + def __init__(self): + self.crud_price = CRUDBase[ + model_CurrencyPrice, CurrencyPrice, CurrencyPriceCreate, CurrencyPriceUpdate + ]( + model=model_CurrencyPrice, + schema=CurrencyPrice, + create_schema=CurrencyPriceCreate, + ) + self.crud_type = CRUDBase[ + model_CurrencyType, CurrencyType, CurrencyTypeCreate, CurrencyTypeUpdate + ]( + model=model_CurrencyType, + schema=CurrencyType, + create_schema=CurrencyTypeCreate, + ) + + self.latest_hours_validate = TypeAdapter(dict[int, int]).validate_python + self.currency_list_validate = TypeAdapter(list[Currency]).validate_python - db_latest_currency_id = db.execute(stmt).mappings().first() - if db_latest_currency_id is not None: - latest_currency_id = db_latest_currency_id["latestCurrencyId"] - else: - latest_currency_id = 1 + async def get_prices(self, *args, **kwargs): + return await self.crud_price.get(*args, **kwargs) - validate = TypeAdapter(int).validate_python + async def get_types(self, *args, **kwargs): + return await self.crud_type.get(*args, **kwargs) - return validate(latest_currency_id) + async def create_prices(self, *args, **kwargs): + return await self.crud_price.create(*args, **kwargs) + + async def create_types(self, *args, **kwargs): + return await self.crud_type.create(*args, **kwargs) + + async def update_type(self, *args, **kwargs): + return await self.crud_type.update(*args, **kwargs) def _latest_hours_stmt(self, league_ids: list[int]): return ( select( - model_Currency.leagueId, - func.max(model_Currency.createdHoursSinceLaunch).label("latest_hour"), + model_CurrencyPrice.leagueId, + func.max(model_CurrencyPrice.createdHoursSinceLaunch).label( + "latest_hour" + ), ) - .where(model_Currency.leagueId.in_(league_ids)) - .group_by(model_Currency.leagueId) + .where(model_CurrencyPrice.leagueId.in_(league_ids)) + .group_by(model_CurrencyPrice.leagueId) ) async def get_latest_hours( @@ -54,50 +74,80 @@ async def get_latest_hours( objs = db.execute(stmt).mappings().all() id_hour_map = {obj["leagueId"]: obj["latest_hour"] for obj in objs} - validate = TypeAdapter(dict[int, int]).validate_python - - return validate(id_hour_map) + return self.latest_hours_validate(id_hour_map) async def get_latest_currencies( self, db: Session, league_ids: list[int] ) -> list[Currency]: latest_hours = self._latest_hours_stmt(league_ids).subquery() - stmt = select(model_Currency).join( - latest_hours, - (model_Currency.leagueId == latest_hours.c.leagueId) - & (model_Currency.createdHoursSinceLaunch == latest_hours.c.latest_hour), + stmt = ( + select(model_CurrencyPrice) + .join( + latest_hours, + (model_CurrencyPrice.leagueId == latest_hours.c.leagueId) + & ( + model_CurrencyPrice.createdHoursSinceLaunch + == latest_hours.c.latest_hour + ), + ) + .join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, + ) ) currencies = db.scalars(stmt).all() - validate = TypeAdapter(list[Currency]).validate_python - - return validate(currencies) + return self.currency_list_validate(currencies) async def get_currency_from_query( - self, db: Session, query_list: list[CurrencyQuery] + self, + db: Session, + query_list: list[CurrencyQuery] | None = None, + filter_params: FilterParams | None = None, ) -> list[Currency]: - filters = [] - for query in query_list: - sub_filter = [] - if query.createdHoursSinceLaunch is not None: - sub_filter.append( - model_Currency.createdHoursSinceLaunch - == query.createdHoursSinceLaunch + stmt = select(model_CurrencyPrice) + + if query_list: + filters = list[ColumnElement[bool]]() + filter_names = False + for query in query_list: + sub_filter = list[ColumnElement[bool]]() + if query.createdHoursSinceLaunch is not None: + sub_filter.append( + model_CurrencyPrice.createdHoursSinceLaunch + == query.createdHoursSinceLaunch + ) + + if query.tradeName is not None: + sub_filter.append(model_CurrencyType.tradeName == query.tradeName) + filter_names = True + + if query.leagueId is not None: + sub_filter.append(model_CurrencyPrice.leagueId == query.leagueId) + + if sub_filter: + filters.append(and_(*sub_filter)) + + if filter_names: + stmt = stmt.join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, ) - if query.tradeName is not None: - sub_filter.append(model_Currency.tradeName == query.tradeName) + stmt = stmt.where(or_(*filters)) - if query.leagueId is not None: - sub_filter.append(model_Currency.leagueId == query.leagueId) - - if sub_filter: - filters.append(and_(*sub_filter)) - stmt = select(model_Currency).where(or_(*filters)) + if filter_params is not None: + if filter_params.skip is not None: + stmt = stmt.offset(filter_params.skip) + if filter_params.limit is not None: + stmt = stmt.limit(filter_params.limit) currencies = db.execute(stmt).scalars().all() - validate = TypeAdapter(list[Currency]).validate_python + if filter_params is not None: + currencies = self.crud_price._sort_objects( + currencies, filter_params.sort_key, filter_params.sort_method + ) - return validate(currencies) + return self.currency_list_validate(currencies) diff --git a/src/backend_api/app/crud/extensions/crud_league.py b/src/backend_api/app/crud/extensions/crud_league.py index 01af9728..e1ce7044 100644 --- a/src/backend_api/app/crud/extensions/crud_league.py +++ b/src/backend_api/app/crud/extensions/crud_league.py @@ -14,6 +14,9 @@ class CRUDLeague(CRUDBase[model_League, League, LeagueCreate, LeagueUpdate]): + def __init__(self): + super().__init__(model=model_League, schema=League, create_schema=LeagueCreate) + async def get_active_leagues(self, db: Session) -> list[League]: now = datetime.now() diff --git a/src/backend_api/app/crud/extensions/crud_modifier.py b/src/backend_api/app/crud/extensions/crud_modifier.py index 9d291ea4..a4dfc3e7 100644 --- a/src/backend_api/app/crud/extensions/crud_modifier.py +++ b/src/backend_api/app/crud/extensions/crud_modifier.py @@ -21,6 +21,13 @@ class CRUDModifier( ModifierUpdate, ] ): + def __init__(self): + super().__init__( + model=model_Modifier, + schema=Modifier, + create_schema=ModifierCreate, + ) + async def get_grouped_modifier_by_effect( self, db: Session ) -> GroupedModifierByEffect: diff --git a/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py b/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py index 3e3ad12e..0be48d04 100644 --- a/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py +++ b/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py @@ -19,6 +19,13 @@ class CRUDUnidentifiedItem( UnidentifiedItemUpdate, ] ): + def __init__(self): + super().__init__( + model=model_UnidentifiedItem, + schema=UnidentifiedItem, + create_schema=UnidentifiedItemCreate, + ) + async def get_non_aggregated(self, db: Session) -> list[UnidentifiedItem]: """ Returns the non aggregated unidentified items diff --git a/src/backend_api/app/plotting/plotter.py b/src/backend_api/app/plotting/plotter.py index 013a88db..33955494 100644 --- a/src/backend_api/app/plotting/plotter.py +++ b/src/backend_api/app/plotting/plotter.py @@ -22,7 +22,10 @@ from app.core.models.database import engine from app.core.models.models import ( - Currency as model_Currency, + CurrencyPrice as model_CurrencyPrice, +) +from app.core.models.models import ( + CurrencyType as model_CurrencyType, ) from app.core.models.models import ( Item as model_Item, @@ -101,17 +104,25 @@ def _init_stmt( item_model.itemBaseTypeId, item_model.currencyId, item_model.currencyAmount, - model_Currency.tradeName, - model_Currency.valueInChaos, - model_Currency.createdHoursSinceLaunch.label( + model_CurrencyType.tradeName, + model_CurrencyPrice.valueInChaos, + model_CurrencyPrice.createdHoursSinceLaunch.label( "currencyCreatedHoursSinceLaunch" ), ] if query_select_args: select_args.extend(query_select_args) - stmt = select(*select_args).join( - model_Currency, item_model.currencyId == model_Currency.currencyId + stmt = ( + select(*select_args) + .join( + model_CurrencyPrice, + item_model.currencyId == model_CurrencyPrice.currencyId, + ) + .join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, + ) ) if isinstance(query.leagueId, list): diff --git a/src/backend_data_retrieval/Dockerfile b/src/backend_data_retrieval/Dockerfile index 5c271593..502757fa 100644 --- a/src/backend_data_retrieval/Dockerfile +++ b/src/backend_data_retrieval/Dockerfile @@ -30,6 +30,13 @@ ENV PYTHONPATH=/data_retrieval_app COPY ./prestart.sh ./pyproject.toml ./uv.lock /data_retrieval_app/ +# For referencing API response schemas +COPY --from=backend_api \ + ./app/core/schemas/item.py \ + ./app/core/schemas/league.py \ + ./app/core/schemas/currency.py \ + /data_retrieval_app/backend_api/app/core/schemas/ + ADD ./data_retrieval_app /data_retrieval_app/data_retrieval_app # Sync the project diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py new file mode 100644 index 00000000..3c857f7b --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py @@ -0,0 +1,176 @@ +from collections.abc import Iterator + +import requests +from backend_api.app.core.schemas.currency import ( + CurrencyType, + CurrencyTypeCreate, + CurrencyTypeUpdate, +) +from pydantic import TypeAdapter + +from data_retrieval_app.data_deposit.data_depositor_base import DataDepositorBase +from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.currency import ( + ExchangeRatioItem, +) +from data_retrieval_app.logs.logger import data_deposit_logger as logger +from data_retrieval_app.pom_api_authentication import get_superuser_token_headers +from data_retrieval_app.utils import get_data_safe + + +class CurrencyTypeDataDepositor(DataDepositorBase): + def __init__(self): + self.base_url = settings.BACKEND_BASE_URL + self.pom_auth_headers = get_superuser_token_headers(self.base_url) + + self.data_url = f"{self.base_url}/currency/type/" + + self.name_to_trade_name = self._get_name_to_trade_name_dict() + + def _get_name_to_trade_name_dict(self) -> dict[str, str]: + """ + Retrieves a map for "fancy" currency names, as used in the API, to their trade names, which we need. + """ + headers = { + "User-Agent": f"OAuth pathofmodifiers/{settings.TAG} (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" + } + response = get_data_safe( + "https://www.pathofexile.com/api/trade/data/static", + headers=headers, + logger=logger, + ) + + response_json = response.json() + result = response_json["result"] + currencies = dict[str, str]() + for category in result: + if category["id"] == "Currency": + for entry in category["entries"]: + name = entry["text"] + trade_name = entry["id"] + currencies[name] = trade_name + + return currencies + + def _get_current_currency_types(self) -> list[CurrencyType]: + response = get_data_safe( + self.data_url, + headers=self.pom_auth_headers, + logger=logger, + ) + + return TypeAdapter(list[CurrencyType]).validate_python(response.json()) + + def _load_data(self) -> Iterator[list[ExchangeRatioItem]]: + response = get_data_safe( + "https://api.poe.watch/exchange/ratios?league=Standard&game=poe1", + logger=logger, + ) + response_json = response.json() + currency_items = list[ExchangeRatioItem]() + currency_item_adaptor = TypeAdapter(ExchangeRatioItem) + for item in response_json["items"]: + if item["category"] == "currency": + item["leagueId"] = -1 + currency_items.append(currency_item_adaptor.validate_python(item)) + + yield currency_items + + def _process_data( + self, currency_items: list[ExchangeRatioItem] + ) -> list[CurrencyTypeCreate]: + """ + Scenarios that requires updated currency: + Name is: + - 'TEMP_NAME': this is right after the 'ee1239y6yfda' migration, + as the database has no knowledge of name + - not the same when trade name is + Trade name is: + - not the same when name is + Incoming currency always takes precedence + """ + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_auth_headers) + + current_currencies = self._get_current_currency_types() + processed_currencies = list[CurrencyTypeCreate]() + for currency in currency_items: + trade_name = self.name_to_trade_name.get(currency.name) + if trade_name is None: + continue + + currency_id = -1 + + found_duplicate = False + need_update = False + for current_currency in current_currencies: + if ( + current_currency.name == "TEMP_NAME" + and current_currency.tradeName == trade_name + ): + need_update = True + if ( + current_currency.name == currency.name + and current_currency.tradeName != trade_name + ): + need_update = True + if ( + current_currency.name != currency.name + and current_currency.tradeName == trade_name + ): + need_update = True + if ( + current_currency.name == currency.name + and current_currency.tradeName == trade_name + ): + found_duplicate = True + + if found_duplicate or need_update: + currency_id = current_currency.currencyId + break + + if need_update: + updated_currency = CurrencyTypeUpdate( + currencyId=currency_id, name=currency.name, tradeName=trade_name + ) + + try: + response = requests.put( + self.data_url, + json=updated_currency.model_dump(), + headers=headers, + ) + response.raise_for_status() + except Exception as e: + logger.error( + f"The following error occurred while inserting data: {e}" + ) + raise e + + elif not found_duplicate: + processed_currencies.append( + CurrencyTypeCreate(name=currency.name, tradeName=trade_name) + ) + + return processed_currencies + + def _insert_data(self, currencies: list[CurrencyTypeCreate]): + if not currencies: + return + currency_create_adapter = TypeAdapter(list[CurrencyTypeCreate]) + + logger.info("Inserting data into database.") + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_auth_headers) + try: + response = requests.post( + self.data_url, + json=currency_create_adapter.dump_python(currencies), + headers=headers, + ) + response.raise_for_status() + except Exception as e: + logger.error(f"The following error occurred while inserting data: {e}") + raise e + + logger.info("Successfully inserted data into database.") diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py index b40c95b1..be85da2f 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py @@ -14,7 +14,10 @@ class DataDepositorBase: - def __init__(self, data_type: Literal["modifier", "item_base_type"]) -> None: + def __init__( + self, + data_type: Literal["modifier", "item_base_type", "league"], + ) -> None: self.base_url = settings.BACKEND_BASE_URL self.pom_auth_headers = get_superuser_token_headers(self.base_url) diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py index 342c0bbc..0dc20f6c 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py @@ -1,3 +1,6 @@ +from data_retrieval_app.data_deposit.currency.currency_type_data_depositor import ( + CurrencyTypeDataDepositor, +) from data_retrieval_app.data_deposit.data_depositor_base import DataDepositorBase from data_retrieval_app.data_deposit.item_base_type.item_base_type_data_depositor import ( ItemBaseTypeDataDepositor, @@ -19,6 +22,7 @@ def main(): "modifer": ModifierDataDepositor(), "itemBaseType": ItemBaseTypeDataDepositor(), "league": LeagueDataDepositor(), + "currencyType": CurrencyTypeDataDepositor(), } for key, data_depositor in data_depositors.items(): logger.info(f"Depositing {key} data.") From c54621aead7c305383120c7b810155476f91fe4d Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 19 Sep 2026 16:37:32 -0230 Subject: [PATCH 05/16] #830 Implemented currency data retrieval --- src/backend_api/app/core/schemas/currency.py | 1 + src/backend_api/app/core/schemas/league.py | 6 +- src/backend_api/app/crud/base.py | 5 +- .../app/crud/extensions/crud_currency.py | 34 +-- .../currency/currency_type_data_depositor.py | 29 ++- .../data_retrieval/currency_api_handler.py | 136 ++++++++++-- .../data_retrieval/poe_api_handler.py | 9 +- .../schemas/external/currency.py | 25 +++ .../{poe_schema.py => external/poe.py} | 0 .../external_data_retrieval/detectors/base.py | 2 +- .../detectors/detector_controller.py | 7 +- .../detectors/unique_detector.py | 2 +- .../external_data_retrieval/main.py | 76 ++----- .../transform_currency_api_data.py | 209 ++++++++---------- .../transform_poe_api_data.py | 23 +- .../data_retrieval_app/utils.py | 36 ++- 16 files changed, 348 insertions(+), 252 deletions(-) create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py rename src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/{poe_schema.py => external/poe.py} (100%) diff --git a/src/backend_api/app/core/schemas/currency.py b/src/backend_api/app/core/schemas/currency.py index 4d9f07ec..97218af3 100644 --- a/src/backend_api/app/core/schemas/currency.py +++ b/src/backend_api/app/core/schemas/currency.py @@ -72,6 +72,7 @@ class CurrencyTypeInDB(CurrencyTypeInDBBase): class Currency(_pydantic.BaseModel): currencyId: int tradeName: str + name: str leagueId: int createdHoursSinceLaunch: int diff --git a/src/backend_api/app/core/schemas/league.py b/src/backend_api/app/core/schemas/league.py index a9e4f900..98853d08 100644 --- a/src/backend_api/app/core/schemas/league.py +++ b/src/backend_api/app/core/schemas/league.py @@ -1,5 +1,3 @@ -import datetime as _dt - import pydantic as _pydantic @@ -8,8 +6,8 @@ class _BaseLeague(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) name: str - validFrom: _dt.datetime - validTo: _dt.datetime | None = None + validFrom: _pydantic.AwareDatetime + validTo: _pydantic.AwareDatetime | None = None version: float diff --git a/src/backend_api/app/crud/base.py b/src/backend_api/app/crud/base.py index f243bcdd..efcce7f8 100644 --- a/src/backend_api/app/crud/base.py +++ b/src/backend_api/app/crud/base.py @@ -55,15 +55,14 @@ def _sort_objects( self, objs: list[ModelType], sort_key: str | None = None, - sort_method: Literal["asc", "dec"] | None = None, + sort_method: Literal["asc", "dec"] = "asc", ) -> list[ModelType]: """ `sort_key` is the column name to sort on. For example `createdAt`. """ if sort_key is None: return objs - if sort_method is None: - sort_method = "asc" + unsorted_extracted_column = [] for obj in objs: unsorted_extracted_column.append(getattr(obj, sort_key)) diff --git a/src/backend_api/app/crud/extensions/crud_currency.py b/src/backend_api/app/crud/extensions/crud_currency.py index 80f7689a..34de3923 100644 --- a/src/backend_api/app/crud/extensions/crud_currency.py +++ b/src/backend_api/app/crud/extensions/crud_currency.py @@ -82,7 +82,14 @@ async def get_latest_currencies( latest_hours = self._latest_hours_stmt(league_ids).subquery() stmt = ( - select(model_CurrencyPrice) + select( + model_CurrencyType.currencyId, + model_CurrencyType.name, + model_CurrencyType.tradeName, + model_CurrencyPrice.leagueId, + model_CurrencyPrice.createdHoursSinceLaunch, + model_CurrencyPrice.valueInChaos, + ) .join( latest_hours, (model_CurrencyPrice.leagueId == latest_hours.c.leagueId) @@ -96,7 +103,7 @@ async def get_latest_currencies( model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, ) ) - currencies = db.scalars(stmt).all() + currencies = db.execute(stmt).mappings().all() return self.currency_list_validate(currencies) @@ -106,11 +113,19 @@ async def get_currency_from_query( query_list: list[CurrencyQuery] | None = None, filter_params: FilterParams | None = None, ) -> list[Currency]: - stmt = select(model_CurrencyPrice) - + stmt = select( + model_CurrencyType.currencyId, + model_CurrencyType.name, + model_CurrencyType.tradeName, + model_CurrencyPrice.leagueId, + model_CurrencyPrice.createdHoursSinceLaunch, + model_CurrencyPrice.valueInChaos, + ).join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, + ) if query_list: filters = list[ColumnElement[bool]]() - filter_names = False for query in query_list: sub_filter = list[ColumnElement[bool]]() if query.createdHoursSinceLaunch is not None: @@ -121,7 +136,6 @@ async def get_currency_from_query( if query.tradeName is not None: sub_filter.append(model_CurrencyType.tradeName == query.tradeName) - filter_names = True if query.leagueId is not None: sub_filter.append(model_CurrencyPrice.leagueId == query.leagueId) @@ -129,12 +143,6 @@ async def get_currency_from_query( if sub_filter: filters.append(and_(*sub_filter)) - if filter_names: - stmt = stmt.join( - model_CurrencyType, - model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, - ) - stmt = stmt.where(or_(*filters)) if filter_params is not None: @@ -143,7 +151,7 @@ async def get_currency_from_query( if filter_params.limit is not None: stmt = stmt.limit(filter_params.limit) - currencies = db.execute(stmt).scalars().all() + currencies = db.execute(stmt).mappings().all() if filter_params is not None: currencies = self.crud_price._sort_objects( diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py index 3c857f7b..fc1c8cf0 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py @@ -15,7 +15,7 @@ ) from data_retrieval_app.logs.logger import data_deposit_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import get_data_safe +from data_retrieval_app.utils import get_data_safe, post_data_safe class CurrencyTypeDataDepositor(DataDepositorBase): @@ -94,6 +94,8 @@ def _process_data( current_currencies = self._get_current_currency_types() processed_currencies = list[CurrencyTypeCreate]() + + found_chaos = False for currency in currency_items: trade_name = self.name_to_trade_name.get(currency.name) if trade_name is None: @@ -125,6 +127,9 @@ def _process_data( ): found_duplicate = True + if current_currency.tradeName == "chaos": + found_chaos = True + if found_duplicate or need_update: currency_id = current_currency.currencyId break @@ -152,25 +157,25 @@ def _process_data( CurrencyTypeCreate(name=currency.name, tradeName=trade_name) ) + if not found_chaos: + processed_currencies.append( + CurrencyTypeCreate(name="Chaos Orb", tradeName="Chaos") + ) + return processed_currencies def _insert_data(self, currencies: list[CurrencyTypeCreate]): if not currencies: return - currency_create_adapter = TypeAdapter(list[CurrencyTypeCreate]) logger.info("Inserting data into database.") headers = {"accept": "application/json", "Content-Type": "application/json"} headers.update(self.pom_auth_headers) - try: - response = requests.post( - self.data_url, - json=currency_create_adapter.dump_python(currencies), - headers=headers, - ) - response.raise_for_status() - except Exception as e: - logger.error(f"The following error occurred while inserting data: {e}") - raise e + + post_data_safe( + self.data_url, + json=TypeAdapter(list[CurrencyTypeCreate]).dump_python(currencies), + headers=headers, + ) logger.info("Successfully inserted data into database.") diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py index a5c1b6e4..f2133290 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py @@ -1,44 +1,142 @@ -from typing import Any +from collections import defaultdict +from collections.abc import Sequence -import pandas as pd +from backend_api.app.core.schemas.currency import Currency, CurrencyType +from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter +from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.currency import ( + ExchangeRatioItem, +) +from data_retrieval_app.external_data_retrieval.transforming_data.transform_currency_api_data import ( + TransformCurrencyAPIData, +) from data_retrieval_app.logs.logger import external_data_retrieval_logger as logger +from data_retrieval_app.pom_api_authentication import ( + get_superuser_token_headers, +) from data_retrieval_app.utils import get_data_safe class CurrencyAPIHandler: + backend_base_url = settings.BACKEND_BASE_URL + pom_auth_headers = get_superuser_token_headers(backend_base_url) + + currency_url = f"{backend_base_url}/currency/" + def __init__(self, url: str) -> None: self.url = url - def _json_to_df(self, currencies: list) -> pd.DataFrame: - df = pd.json_normalize(currencies) + self.currency_item_adapter = TypeAdapter(ExchangeRatioItem) + + self.currency_types = self._get_currency_types() + self.transformer = TransformCurrencyAPIData(self.currency_types) + + def _get_currency_types(self) -> dict[str, CurrencyType]: + response = get_data_safe( + self.currency_url + "type/", + headers=self.pom_auth_headers, + logger=logger, + ) + + currency_types = TypeAdapter(list[CurrencyType]).validate_python( + response.json() + ) + return {currency_type.name: currency_type for currency_type in currency_types} + + def _get_latest_currencies( + self, league_ids: Sequence[int] + ) -> dict[str, list[Currency]]: + response = get_data_safe( + self.currency_url + "price/latest/", + params={"league_ids": league_ids}, + headers=self.pom_auth_headers, + logger=logger, + ) + currencies = TypeAdapter(list[Currency]).validate_python(response.json()) - return df + trade_name_to_currencies = defaultdict[str, list[Currency]](list) + for currency in currencies: + trade_name_to_currencies[currency.tradeName].append(currency) - def make_request(self, leagues: list[dict[str, Any]]) -> pd.DataFrame: + return dict(trade_name_to_currencies) + + def _remove_old_currencies( + self, + trade_name_to_currencies: dict[str, list[Currency]], + leagues: list[League], + current_hours: dict[int, int], + ) -> list[League]: """ - Makes an initial, synchronous, API call. + Modifies latest currencies inplace """ - df = None + leagues_needs_new_data = list[League]() + if not trade_name_to_currencies: + # No prior data is available + return leagues + + for currencies in trade_name_to_currencies.values(): + for league in leagues: + for currency in currencies[:]: + if league.leagueId != currency.leagueId: + continue + + current_hour = current_hours[currency.leagueId] + if currency.createdHoursSinceLaunch == current_hour: + continue + + currencies.remove(currency) + + if league not in leagues_needs_new_data: + leagues_needs_new_data.append(league) + + return leagues_needs_new_data + + def _get_new_data(self, leagues: list[League]) -> list[ExchangeRatioItem]: + currency_items = list[ExchangeRatioItem]() for league in leagues: response = get_data_safe( - self.url.format(league=league["name"].replace(" ", "+")), logger=logger + self.url.format(league=league.name.replace(" ", "+")), logger=logger ) response_json = response.json() - items_df = pd.json_normalize(response_json["items"]) - currency_df = items_df[items_df["category"] == "currency"] - currency_df["leagueId"] = league["leagueId"] - if df is None: - df = currency_df - else: - df = pd.concat((df, currency_df)) - return df + for item in response_json["items"]: + if item["category"] == "currency": + item["leagueId"] = league.leagueId + currency_items.append( + self.currency_item_adapter.validate_python(item) + ) + + return currency_items + + def get_currency_data( + self, leagues: list[League], current_hours: dict[int, int] + ) -> dict[str, list[Currency]]: + """ + Returns a dict mapping trade name to list of the same currency per league + """ + id_to_league = {league.leagueId: league for league in leagues} + trade_name_to_currencies = self._get_latest_currencies(id_to_league.keys()) + print(trade_name_to_currencies) + + leagues_needs_new_data = self._remove_old_currencies( + trade_name_to_currencies, leagues, current_hours + ) + + if leagues_needs_new_data: + exchange_ratios = self._get_new_data(leagues_needs_new_data) + trade_name_to_currencies.update( + self.transformer.transform_and_insert(exchange_ratios, current_hours) + ) + + return dict(trade_name_to_currencies) def store_data_to_csv(self, path: str) -> None: """ Stores the data in a CSV. Only to be used for testing purposes. """ - currencies_df = self.make_request() + # currencies_df = self.make_request() - currencies_df.to_csv(path + "/currencies.csv", index=False) + # currencies_df.to_csv(path + "/currencies.csv", index=False) + raise NotImplementedError() diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py index cbc68433..742007a6 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py @@ -10,7 +10,9 @@ import httpx import pandas as pd import redis +from backend_api.app.core.schemas.league import League +from data_retrieval_app.external_data_retrieval.cache import get_cache from data_retrieval_app.external_data_retrieval.config import settings from data_retrieval_app.external_data_retrieval.data_retrieval.utils import ( ByteResponse, @@ -50,7 +52,7 @@ def __init__( url: str, auth_token: str, *, - leagues: list[dict[str, Any]], + leagues: list[League], detector_controller: DetectorController | None = None, ) -> None: """ @@ -71,6 +73,7 @@ def __init__( UniqueUnidentifiedDetector(), ], leagues, + get_cache(), ) self.url = url logger.debug("Url set to: " + self.url) @@ -132,9 +135,7 @@ def _follow_stream( # pick up from latest checkpoint if listener_id == 0: logger.debug("Main listener initiating the ping-pong again") - change_id = cache.get( - f"next_change_id:{self.leagues[0]["name"]}" - ) + change_id = cache.get(f"next_change_id:{self.leagues[0].name}") if change_id is None: change_id = self.initial_change_id # Make sure second listener also resets diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py new file mode 100644 index 00000000..fe1d846c --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + + +class CurrencyModel(BaseModel): + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + ) + + +class ExchangeRatioSide(BaseModel): + chaosValue: float | None = None + + +class ExchangeRatioItem(BaseModel): + id: int + + name: str + category: str + leagueId: int # supplied internally from DB + + chaos: ExchangeRatioSide + divine: ExchangeRatioSide diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/poe_schema.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py similarity index 100% rename from src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/poe_schema.py rename to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py index c0ec5acf..de6b006e 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.poe_schema import ( +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( Item, Stash, ) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py index c5a37eed..cef165b6 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py @@ -1,9 +1,10 @@ from typing import Any import redis +from backend_api.app.core.schemas.league import League from pydantic import TypeAdapter -from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.poe_schema import ( +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( CacheItem, CacheStash, Item, @@ -16,10 +17,10 @@ class DetectorController: def __init__( self, detectors: list[DetectorBase], - leagues: list[dict[str, Any]], + leagues: list[League], redis_cache: redis.Redis, ): - self.leagues = [league["name"] for league in leagues] + self.leagues = [league.name for league in leagues] self.detectors = detectors diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py index d2c6eab5..caca95ac 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py @@ -1,4 +1,4 @@ -from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.poe_schema import ( +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( Item, ) from data_retrieval_app.external_data_retrieval.detectors.base import DetectorBase diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py index 9dc4d74c..aa57a2ab 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py @@ -7,10 +7,11 @@ wait, ) from io import StringIO -from typing import Any import pandas as pd import redis +from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter from data_retrieval_app.external_data_retrieval.cache import get_cache from data_retrieval_app.external_data_retrieval.config import settings @@ -20,9 +21,6 @@ from data_retrieval_app.external_data_retrieval.data_retrieval.poe_api_handler import ( PoEAPIHandler, ) -from data_retrieval_app.external_data_retrieval.transforming_data.transform_currency_api_data import ( - TransformCurrencyAPIData, -) from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_api_data import ( PoEAPIDataTransformerBase, UniquePoEAPIDataTransformer, @@ -68,7 +66,6 @@ def __init__( self.currency_api_handler = CurrencyAPIHandler( url="https://api.poe.watch/exchange/ratios?league={league}&game=poe1" ) - self.currency_transformer = TransformCurrencyAPIData() def _get_modifiers(self) -> dict[str, pd.DataFrame]: response = get_data_safe( @@ -99,13 +96,12 @@ def _get_modifiers(self) -> dict[str, pd.DataFrame]: ] return modifier_dfs - def _get_leagues(self) -> list[dict[str, Any]]: + def _get_leagues(self) -> list[League]: response = get_data_safe( self.active_league_url, headers=self.pom_auth_headers, logger=logger ) - leagues = response.json() - return leagues + return TypeAdapter(list[League]).validate_python(response.json()) def _get_item_base_types(self) -> dict[str, int]: response = get_data_safe( @@ -152,66 +148,20 @@ def _categorize_new_items(self, df: pd.DataFrame) -> dict[str, pd.DataFrame]: return split_dfs - def _get_currency_data(self, current_hours: dict[int, int]) -> pd.DataFrame: - league_ids = list(current_hours.keys()) - response = get_data_safe( - self.currency_url + "latest_hours/", - params={"league_ids": league_ids}, - headers=self.pom_auth_headers, - logger=logger, - ) - - latest_hours: dict[str, int] = response.json() - need_new_data = [] - need_old_data = [] - if latest_hours: - for league_id, latest_hour in latest_hours.items(): - league_id = int(league_id) - if ( - league_id in current_hours - and latest_hour == current_hours[league_id] - ): - need_old_data.append(league_id) - else: - need_new_data.append(league_id) - else: - need_new_data = league_ids - - currency_df = None - if need_old_data: - response = get_data_safe( - self.currency_url + "latest_currencies/", - params={"league_ids": need_old_data}, - headers=self.pom_auth_headers, - logger=logger, - ) - - currency_df = pd.DataFrame(response.json()) - - if need_new_data: - needed_leagues = [ - league for league in self.leagues if league["leagueId"] in need_new_data - ] - new_data = self.currency_api_handler.make_request(needed_leagues) - new_data = self.currency_transformer.transform_into_tables( - new_data, current_hours - ) - if currency_df is None: - currency_df = new_data - else: - currency_df = pd.concat((currency_df, new_data)) - - return currency_df - def _follow_data_dump_stream(self, cache: redis.Redis): current_hours = find_hours_since_launch(self.leagues) # Only need to refer to one league to see when a new hour starts - current_hour = current_hours[self.leagues[0]["leagueId"]] + current_hour = current_hours[self.leagues[0].leagueId] next_hour = current_hour + 1 logger.info("Retrieving modifiers from db.") modifier_dfs = self._get_modifiers() item_base_types = self._get_item_base_types() - currency_df = self._get_currency_data(current_hours) + print(self.leagues) + trade_name_to_currencies = self.currency_api_handler.get_currency_data( + self.leagues, current_hours + ) + print(trade_name_to_currencies) + exit() iter_data = self.poe_api_handler.dump_stream() while current_hour < next_hour: df, next_change_id = next(iter_data) @@ -222,7 +172,7 @@ def _follow_data_dump_stream(self, cache: redis.Redis): self.data_transformers[data_transformer_type].transform_into_tables( df=split_dfs[data_transformer_type], modifier_df=modifier_dfs[data_transformer_type], - currency_df=currency_df.copy(deep=True), + # currency_df=currency_df.copy(deep=True), item_base_types=item_base_types, current_hours=current_hours, ) @@ -231,7 +181,7 @@ def _follow_data_dump_stream(self, cache: redis.Redis): cache.set("next_change_id", next_change_id) current_hours = find_hours_since_launch(self.leagues) - current_hour = current_hours[self.leagues[0]["leagueId"]] + current_hour = current_hour = current_hours[self.leagues[0].leagueId] for data_transformer_type in self.data_transformers: self.data_transformers[data_transformer_type].end_of_hour_cleanup() diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py index 8465d070..c24fcc57 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py @@ -1,138 +1,113 @@ -import pandas as pd +from collections import defaultdict + +from backend_api.app.core.schemas.currency import ( + Currency, + CurrencyPriceCreate, + CurrencyType, +) +from pydantic import TypeAdapter from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.currency import ( + ExchangeRatioItem, +) from data_retrieval_app.logs.logger import transform_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import get_data_safe, insert_data +from data_retrieval_app.utils import post_data_safe class TransformCurrencyAPIData: - def __init__(self) -> None: + def __init__(self, name_to_currency: dict[str, CurrencyType]) -> None: logger.debug("Initializing TransformCurrencyAPIData.") self.base_url = settings.BACKEND_BASE_URL - logger.debug(f"Url set to: {self.base_url}") + self.url = f"{self.base_url}/currency/price/" + logger.debug(f"Url set to: {self.url}") self.pom_api_headers = get_superuser_token_headers(self.base_url) logger.debug("Headers set to: " + str(self.pom_api_headers)) logger.debug("Initializing TransformCurrencyAPIData done.") - self.name_to_trade_name = self._get_name_to_trade_name_dict() + self.name_to_currency = name_to_currency - def _get_name_to_trade_name_dict(self) -> dict: - """ - Retrieves a map for "fancy" currency names, as used in the API, to their trade names, which we need. - """ - headers = { - "User-Agent": f"OAuth pathofmodifiers/0.1.0 (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" - } - response = get_data_safe( - "https://www.pathofexile.com/api/trade/data/static", - headers=headers, - logger=logger, - ) - - response_json = response.json() - result = response_json["result"] - currencies = {} - for category in result: - if category["id"] == "Currency": - for entry in category["entries"]: - name = entry["text"] - trade_name = entry["id"] - currencies[name] = trade_name - - return currencies - - def _transform_currency_table( - self, currency_df: pd.DataFrame, current_hours: dict[int, int] - ) -> pd.DataFrame: + def _transform( + self, + exchange_ratios: list[ExchangeRatioItem], + current_hours: dict[int, int], + ) -> dict[str, Currency]: """ Since a chaos orb is always worth one chaos orb, ninja does not include it in its price api. """ - - missing_chaos_value_mask = (currency_df["chaos.chaosValue"] == 0) | ( - currency_df["chaos.chaosValue"].isna() - ) - currency_df["chaos.chaosValue"] = currency_df["chaos.chaosValue"].where( - ~missing_chaos_value_mask, - currency_df["divine.chaosValue"], - ) - - chaos_dict = { - "name": ["Chaos Orb"], - "chaos.chaosValue": [1], - } - for league_id in currency_df["leagueId"].unique(): - chaos_dict["leagueId"] = [league_id] - chaos_df = pd.DataFrame.from_dict(chaos_dict) - currency_df = pd.concat((currency_df, chaos_df), ignore_index=True) - - currency_df["tradeName"] = currency_df["name"].map( - lambda name: self.name_to_trade_name.get(name, pd.NA) - ) - - currency_df["createdHoursSinceLaunch"] = currency_df["leagueId"].map( - current_hours - ) - return currency_df - - def _clean_currency_table(self, currency_df: pd.DataFrame) -> pd.DataFrame: - """ - Cleans the currency table of unnecessary columns. - """ - currency_df = currency_df.rename(columns={"chaos.chaosValue": "valueInChaos"}) - - currency_df = currency_df.drop( - currency_df.columns.difference( - ["tradeName", "valueInChaos", "createdHoursSinceLaunch", "leagueId"] - ), - axis=1, - ) - currency_df = currency_df.loc[~currency_df["tradeName"].isna()].reset_index( - drop=True - ) - return currency_df - - def _get_latest_currency_id_series(self, currency_df: pd.DataFrame) -> pd.Series: - response = get_data_safe( - f"{self.base_url}/currency/latest_currency_id/", - headers=self.pom_api_headers, - logger=logger, - ) - latest_currency_id = int(response.text) - - currency_id = pd.Series( - range(latest_currency_id - len(currency_df) + 1, latest_currency_id + 1), - dtype=int, + trade_name_to_currencies = defaultdict[str, list[Currency]](list) + for ratio in exchange_ratios: + currency_type = self.name_to_currency.get(ratio.name) + if currency_type is None: + continue + + value = -1 + if ratio.chaos.chaosValue is None or ratio.chaos.chaosValue == 0: + value = ratio.divine.chaosValue + else: + value = ratio.chaos.chaosValue + + current_hour = current_hours[ratio.leagueId] + currency_id = currency_type.currencyId + trade_name = currency_type.tradeName + + currency = Currency( + currencyId=currency_id, + tradeName=trade_name, + name=ratio.name, + leagueId=ratio.leagueId, + createdHoursSinceLaunch=current_hour, + valueInChaos=value, + ) + + trade_name_to_currencies[ratio.leagueId].append(currency) + + for league_id, current_hour in current_hours.items(): + name = "Chaos Orb" + currency_type = self.name_to_currency[name] + currency_id = currency_type.currencyId + trade_name = currency_type.tradeName + chaos_currency = Currency( + currencyId=currency_id, + tradeName=trade_name, + name=name, + leagueId=league_id, + createdHoursSinceLaunch=current_hour, + valueInChaos=1, + ) + trade_name_to_currencies[league_id].append(chaos_currency) + + return dict(trade_name_to_currencies) + + def _insert(self, trade_name_to_currencies: dict[int, list[Currency]]): + prices = list[CurrencyPriceCreate]() + for currencies in trade_name_to_currencies.values(): + for currency in currencies: + prices.append( + CurrencyPriceCreate( + currencyId=currency.currencyId, + leagueId=currency.leagueId, + createdHoursSinceLaunch=currency.createdHoursSinceLaunch, + valueInChaos=currency.valueInChaos, + ) + ) + + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_api_headers) + post_data_safe( + self.url, + json=TypeAdapter(list[CurrencyPriceCreate]).dump_python(prices), + headers=headers, ) - return currency_id - def transform_into_tables( - self, currency_df: pd.DataFrame, current_hours: dict[int, int] - ) -> pd.DataFrame: - """ - Transforms the data into tables and transforms with help functions. - """ - logger.debug("Transforming data into tables.") - currency_df = self._transform_currency_table(currency_df, current_hours) - logger.debug("Successfully transformed data into tables.") - - logger.debug("Cleaning currency table data.") - currency_df = self._clean_currency_table(currency_df) - logger.debug("Successfully cleaned currency table data.") - - logger.debug("Inserting currency data into database.") - insert_data( - currency_df, - url=self.base_url, - table_name="currency", - logger=logger, - headers=self.pom_api_headers, - ) + def transform_and_insert( + self, exchange_ratios: list[ExchangeRatioItem], current_hours: dict[int, int] + ) -> dict[int, list[Currency]]: + logger.debug("Transforming exchange ratios into currencies.") + trade_name_to_currencies = self._transform(exchange_ratios, current_hours) + logger.debug("Inserting currency prices.") + self._insert(trade_name_to_currencies) logger.debug("Successfully inserted currency data into database.") - currency_id = self._get_latest_currency_id_series(currency_df) - logger.debug("Latest currency id found: " + str(currency_id)) - - currency_df = currency_df.assign(currencyId=currency_id) - logger.debug("Successfully transformed data into tables.") - return currency_df + return trade_name_to_currencies diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py index 6f4700f9..8e0b8917 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py @@ -1,6 +1,5 @@ -from typing import Any - import pandas as pd +from backend_api.app.core.schemas.league import League from requests.exceptions import HTTPError from data_retrieval_app.external_data_retrieval.config import settings @@ -18,7 +17,7 @@ class PoEAPIDataTransformerBase: def __init__( self, - leagues: list[dict[str, Any]], + leagues: list[League], ) -> None: logger.debug("Initializing PoEAPIDataTransformer") @@ -26,7 +25,7 @@ def __init__( self.pom_auth_headers = get_superuser_token_headers(self.base_url) self.roll_processor = RollProcessor() - self.league_to_id = {league["name"]: league["leagueId"] for league in leagues} + self.league_to_id = {league.name: league.leagueId for league in leagues} logger.debug("Initializing PoEAPIDataTransformer done.") @@ -93,6 +92,22 @@ def _find_not_too_highly_priced_item_mask( return ~currency_too_high_mask + # @staticmethod + # def extract_price(item_note: str, stash_note: str) -> tuple[float, str] | None: + # """If the item has a price, it returns the price and currency. Otherwise None is returned""" + + # def has_price(note: str): + # return note is not None and note.startswith(("~b/o", "~price")) + + # if has_price(item_note): + # _, price_str, currency = item_note.split(" ") + # elif has_price(stash_note): + # _, price_str, currency = stash_note.split(" ") + # else: + # return None + + # return float(price_str), currency + @sync_timing_tracker def _transform_item_table( self, diff --git a/src/backend_data_retrieval/data_retrieval_app/utils.py b/src/backend_data_retrieval/data_retrieval_app/utils.py index 0e0d6f89..a5063178 100644 --- a/src/backend_data_retrieval/data_retrieval_app/utils.py +++ b/src/backend_data_retrieval/data_retrieval_app/utils.py @@ -5,6 +5,7 @@ import pandas as pd import requests +from backend_api.app.core.schemas.league import League from pydantic import HttpUrl from data_retrieval_app.logs.logger import main_logger as logger @@ -53,23 +54,21 @@ def df_to_JSON( ) -def find_hours_since_launch(leagues_df: list[dict]) -> dict[int, int]: +def find_hours_since_launch(leagues: list[League]) -> dict[int, int]: """ Finds the number of hours since launch for each of the leagues in the given dataframe """ current_time = datetime.now(UTC) - hours_since_launch_dict = {} - for league in leagues_df: - league_launch_time = datetime.fromisoformat(league["validFrom"]) - - time_since_launch = current_time - league_launch_time + hours_since_launch_dict = dict[int, int]() + for league in leagues: + time_since_launch = current_time - league.validFrom days_since_launch, seconds_since_launch = ( time_since_launch.days, time_since_launch.seconds, ) hours_since_launch = days_since_launch * 24 + seconds_since_launch // 3600 - hours_since_launch_dict[league["leagueId"]] = hours_since_launch + hours_since_launch_dict[league.leagueId] = hours_since_launch return hours_since_launch_dict @@ -142,7 +141,28 @@ def get_data_safe( except Exception as e: if logger is not None: logger.error( - f"The following error occurred while making request a request to {url}: {e}" + f"The following error occurred while making request a get request to {url}: {e}" + ) + raise e + + return response + + +def post_data_safe( + url: str, + *, + json: dict, + logger: logging.Logger = None, + params: dict | list = None, + headers: dict[str, str] | None = None, +) -> requests.Response: + try: + response = requests.post(url, json=json, params=params, headers=headers) + response.raise_for_status() + except Exception as e: + if logger is not None: + logger.error( + f"The following error occurred while making request a post request to {url}: {e}" ) raise e From d00d842bfae89f15532f711479f42ee7f2b31b57 Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 26 Sep 2026 17:16:24 -0230 Subject: [PATCH 06/16] #830 Added all api schemas and added context information to docker compose files --- src/backend_data_retrieval/Dockerfile | 4 +--- src/docker-compose.override.yml | 5 +++++ src/docker-compose.yml | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend_data_retrieval/Dockerfile b/src/backend_data_retrieval/Dockerfile index 502757fa..f2cc6b43 100644 --- a/src/backend_data_retrieval/Dockerfile +++ b/src/backend_data_retrieval/Dockerfile @@ -32,9 +32,7 @@ COPY ./prestart.sh ./pyproject.toml ./uv.lock /data_retrieval_app/ # For referencing API response schemas COPY --from=backend_api \ - ./app/core/schemas/item.py \ - ./app/core/schemas/league.py \ - ./app/core/schemas/currency.py \ + ./app/core/schemas/ \ /data_retrieval_app/backend_api/app/core/schemas/ ADD ./data_retrieval_app /data_retrieval_app/data_retrieval_app diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index 02a72fae..5adbb8b3 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -92,6 +92,8 @@ services: - "8889:8889" build: context: ./backend_data_retrieval + additional_contexts: + backend_api: ./backend_api args: INSTALL_DEV: ${INSTALL_DEV-true} command: /data_retrieval_app/prestart.sh @@ -105,6 +107,9 @@ services: - uv.lock - path: ./backend_data_retrieval/pyproject.toml action: rebuild + - path: ./backend_api/app/core/schemas + action: sync+restart + target: /data_retrieval_app/backend_api/app/core/schemas frontend: restart: "no" diff --git a/src/docker-compose.yml b/src/docker-compose.yml index ee76fc8c..9e01a867 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -121,6 +121,8 @@ services: - .env build: context: ./backend_data_retrieval + additional_contexts: + backend_api: ./backend_api args: INSTALL_DEV: ${INSTALL_DEV-false} From 61c15c06073ff7005f5e4e4e9025a9f13fcfb83b Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 26 Sep 2026 17:25:32 -0230 Subject: [PATCH 07/16] #830 Split modifier into two, reworked some endpoints and data deposit --- .../23d7asdb13df_split_modifier_table.py | 178 +++++++++ .../dd4abb2b6541_optimize_item_table.py | 5 + src/backend_api/app/api/api.py | 14 - src/backend_api/app/api/routes/__init__.py | 10 +- .../app/api/routes/item_modifier.py | 60 --- src/backend_api/app/api/routes/modifier.py | 37 +- src/backend_api/app/core/models/models.py | 74 ++-- src/backend_api/app/core/schemas/__init__.py | 17 +- .../app/core/schemas/item_modifier.py | 28 +- src/backend_api/app/core/schemas/modifier.py | 71 ++-- src/backend_api/app/crud/__init__.py | 19 +- .../app/crud/extensions/crud_modifier.py | 108 ++++-- .../currency/currency_type_data_depositor.py | 25 +- .../modifier/modifier_data_depositor.py | 364 +++++++++--------- .../modifier/modifier_processing_modules.py | 340 +--------------- .../data_retrieval_app/utils.py | 30 +- 16 files changed, 598 insertions(+), 782 deletions(-) create mode 100644 src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py delete mode 100644 src/backend_api/app/api/routes/item_modifier.py diff --git a/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py new file mode 100644 index 00000000..1895989d --- /dev/null +++ b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py @@ -0,0 +1,178 @@ +"""optimize item table + +Revision ID: 23d7asdb13df +Revises: dd4abb2b6541 +Create Date: 2026-07-23 15:20:18.565656 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from app.alembic.replaceable_objects.main import ReplaceableTrigger + +# revision identifiers, used by Alembic. +revision: str = "23d7asdb13df" +down_revision: Union[str, None] = "dd4abb2b6541" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +modifier_id_trigger = ReplaceableTrigger( + "increment_modifier_id", + "modifier", + """ + RETURNS trigger AS ${name}$ + DECLARE + exists boolean; + BEGIN + exists := EXISTS(SELECT 1 FROM modifier WHERE "effect" = NEW.effect); + IF NOT exists THEN + NEW."modifierId" := nextval('modifier_id_seq'); + + ELSIF exists THEN + NEW."modifierId" := (SELECT "modifierId" FROM modifier WHERE "effect" = NEW.effect LIMIT 1); + END IF; + + RETURN NEW; + END; + ${name}$ LANGUAGE plpgsql; + """, + """ + BEFORE INSERT ON {table} + FOR EACH ROW + EXECUTE FUNCTION {name}(); + """, +) + + +def upgrade() -> None: + op.drop_trigger(modifier_id_trigger) + + op.execute(""" + DROP SEQUENCE modifier_id_seq; + """) + + op.create_table( + "modifier_roll", + sa.Column("modifierId", sa.SmallInteger(), nullable=False), + sa.Column("position", sa.SmallInteger(), nullable=False), + sa.Column("minRoll", sa.Float(4)), + sa.Column("maxRoll", sa.Float(4)), + sa.Column("textRolls", sa.ARRAY(sa.Text())), + sa.PrimaryKeyConstraint("modifierId", "position"), + sa.CheckConstraint( + """ modifier_roll."maxRoll" >= modifier_roll."minRoll" """, + name="check_modifier_maxRoll_greaterThan_minRoll", + ), + ) + + op.execute(""" + INSERT INTO modifier_roll + SELECT + m."modifierId", + m.position, + m."minRoll", + m."maxRoll", + string_to_array(m."textRolls", '|') + FROM modifier m + """) + + with op.batch_alter_table("item_modifier") as batch_op: + batch_op.drop_constraint("fk_item_modifier_modifier", type_="foreignkey") + batch_op.create_foreign_key( + "fk_item_modifier_modifier_roll", + "modifier_roll", + ["modifierId", "position"], + ["modifierId", "position"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + + with op.batch_alter_table("modifier") as batch_op: + batch_op.drop_constraint("modifier_pkey", type_="primary") + batch_op.execute(""" + DELETE FROM modifier m + WHERE m.position > 0 + """) + batch_op.drop_column("position") + batch_op.drop_column("minRoll") + batch_op.drop_column("maxRoll") + batch_op.drop_column("textRolls") + batch_op.create_primary_key("pk_modifier", ["modifierId"]) + batch_op.create_unique_constraint("uq_effect") + + with op.batch_alter_table("modifier_roll") as batch_op: + batch_op.create_foreign_key( + "fk_modifier_roll_modifier", + "modifier", + ["modifierId"], + ["modifierId"], + ondelete="CASCADE", + onupdate="CASCADE", + ) + + +def downgrade() -> None: + # Remove the FK from modifier_roll -> modifier first. + with op.batch_alter_table("modifier_roll") as batch_op: + batch_op.drop_constraint( + "fk_modifier_roll_modifier", + type_="foreignkey", + ) + + with op.batch_alter_table("modifier") as batch_op: + batch_op.drop_constraint("uq_effect") + batch_op.add_column(sa.Column("position", sa.SmallInteger())) + batch_op.add_column(sa.Column("minRoll", sa.Float(4))) + batch_op.add_column(sa.Column("maxRoll", sa.Float(4))) + batch_op.add_column(sa.Column("textRolls", sa.Text())) + + batch_op.create_primary_key( + "modifier_pkey", + ["modifierId", "position"], + ) + + # Restore the rows that were deleted during upgrade and populate + # the position 0 rows with the data from modifier_roll. + op.execute(""" + INSERT INTO modifier ( + "modifierId", + position, + "minRoll", + "maxRoll", + "textRolls" + ) + SELECT + mr."modifierId", + mr.position, + mr."minRoll", + mr."maxRoll", + array_to_string(mr."textRolls", '|') + FROM modifier_roll mr + """) + + # Restore the original FK from item_modifier -> modifier. + with op.batch_alter_table("item_modifier") as batch_op: + batch_op.drop_constraint( + "fk_item_modifier_modifier_roll", + type_="foreignkey", + ) + batch_op.create_foreign_key( + "fk_item_modifier_modifier", + "modifier", + ["modifierId"], + ["modifierId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + + # modifier_roll is no longer needed. + op.drop_table("modifier_roll") + + op.execute("CREATE SEQUENCE modifier_id_seq;") + op.execute( + """SELECT setval('modifier_id_seq', (SELECT MAX("modifierId") FROM modifier));""" + ) + op.create_trigger(modifier_id_trigger) diff --git a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py index df3df289..01fac0eb 100644 --- a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py +++ b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py @@ -206,6 +206,10 @@ def upgrade() -> None: ["leagueId", "itemBaseTypeId", "firstObserved"], ) + batch_op.create_unique_constraint( + "uq_item_game_item_id_league_id", ["gameItemId", "leagueId"] + ) + op.create_table( "item_availability", sa.Column( @@ -235,6 +239,7 @@ def upgrade() -> None: ), # TODO optimize when writing the plotting query sa.Index("ix_item_availability_itemId_validFrom", "itemId", "validFrom"), + sa.UniqueConstraint("itemId", "validFrom"), ) op.execute(""" INSERT INTO item_availability ( diff --git a/src/backend_api/app/api/api.py b/src/backend_api/app/api/api.py index 545d49a7..81236ed1 100644 --- a/src/backend_api/app/api/api.py +++ b/src/backend_api/app/api/api.py @@ -4,12 +4,8 @@ currency, currency_prefix, item, - item_availability, - item_availability_prefix, item_base_type, item_base_type_prefix, - item_modifier, - item_modifier_prefix, item_prefix, league, league_prefix, @@ -33,21 +29,11 @@ api_router.include_router( currency.router, prefix=f"/{currency_prefix}", tags=[currency_prefix] ) -api_router.include_router( - item_availability.router, - prefix=f"/{item_availability_prefix}", - tags=[item_availability_prefix], -) api_router.include_router( item_base_type.router, prefix=f"/{item_base_type_prefix}", tags=[item_base_type_prefix], ) -api_router.include_router( - item_modifier.router, - prefix=f"/{item_modifier_prefix}", - tags=[item_modifier_prefix], -) api_router.include_router(item.router, prefix=f"/{item_prefix}", tags=[item_prefix]) api_router.include_router( league.router, prefix=f"/{league_prefix}", tags=[league_prefix] diff --git a/src/backend_api/app/api/routes/__init__.py b/src/backend_api/app/api/routes/__init__.py index 2f62f675..ea1e0336 100644 --- a/src/backend_api/app/api/routes/__init__.py +++ b/src/backend_api/app/api/routes/__init__.py @@ -1,13 +1,11 @@ from app.api.routes.currency import currency_prefix -from app.api.routes.item_availability import item_availability_prefix -from app.api.routes.item_base_type import item_base_type_prefix -from app.api.routes.item_modifier import item_modifier_prefix from app.api.routes.item import item_prefix -from app.api.routes.unidentified_item import unidentified_item_prefix +from app.api.routes.item_base_type import item_base_type_prefix +from app.api.routes.league import league_prefix from app.api.routes.login import login_prefix from app.api.routes.modifier import modifier_prefix from app.api.routes.plot import plot_prefix +from app.api.routes.test import test_prefix from app.api.routes.turnstile import turnstile_prefix +from app.api.routes.unidentified_item import unidentified_item_prefix from app.api.routes.user import user_prefix -from app.api.routes.test import test_prefix -from app.api.routes.league import league_prefix diff --git a/src/backend_api/app/api/routes/item_modifier.py b/src/backend_api/app/api/routes/item_modifier.py deleted file mode 100644 index 92154a3c..00000000 --- a/src/backend_api/app/api/routes/item_modifier.py +++ /dev/null @@ -1,60 +0,0 @@ -from typing import Annotated - -from fastapi import APIRouter, Depends, Query -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( - get_current_active_superuser, - get_db, -) -from app.api.params import FilterParams -from app.crud import CRUD_itemModifier - -router = APIRouter() - - -item_modifier_prefix = "itemModifier" - - -@router.get( - "/", - response_model=schemas.ItemModifier | list[schemas.ItemModifier], - dependencies=[Depends(get_current_active_superuser)], -) -async def get_all_item_modifiers( - filter_params: Annotated[FilterParams, Query()], - db: Session = Depends(get_db), -): - """ - Get all item modifiers. - - Returns a list of all item modifiers. - """ - - all_itemModifiers = await CRUD_itemModifier.get(db=db, filter_params=filter_params) - - return all_itemModifiers - - -@router.post( - "/", - response_model=schemas.ItemModifierCreate | list[schemas.ItemModifierCreate] | None, - dependencies=[Depends(get_current_active_superuser)], -) -async def create_item_modifier( - itemModifier: schemas.ItemModifierCreate | list[schemas.ItemModifierCreate], - return_nothing: bool | None = None, - db: Session = Depends(get_db), -): - """ - Create one or a list item modifiers. - - Returns the created item modifier or list of item modifiers. - """ - - return await CRUD_itemModifier.create( - db=db, - obj_in=itemModifier, - return_nothing=return_nothing, - ) diff --git a/src/backend_api/app/api/routes/modifier.py b/src/backend_api/app/api/routes/modifier.py index cd83a6e2..4d62967c 100644 --- a/src/backend_api/app/api/routes/modifier.py +++ b/src/backend_api/app/api/routes/modifier.py @@ -88,9 +88,8 @@ async def get_all_modifiers( @router.get( - "/grouped_modifiers_by_effect/", - response_model=schemas.GroupedModifierByEffect - | list[schemas.GroupedModifierByEffect], + "/grouped/", + response_model=list[schemas.GroupedModifier], ) @apply_ip_rate_limits( rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, @@ -109,43 +108,33 @@ async def get_grouped_modifier_by_effect( Returns a list of all grouped modifiers by effect. """ - all_grouped_modifiers_by_effect = ( - await CRUD_modifier.get_grouped_modifier_by_effect(db=db) - ) + all_grouped_modifiers_by_effect = await CRUD_modifier.get_grouped_modifiers(db=db) return all_grouped_modifiers_by_effect @router.post( "/", - response_model=schemas.ModifierCreate | list[schemas.ModifierCreate] | None, + response_model=list[schemas.Modifier], dependencies=[Depends(get_current_active_superuser)], ) async def create_modifier( - modifier: schemas.ModifierCreate | list[schemas.ModifierCreate], - return_nothing: bool | None = None, + modifiers: list[schemas.ModifierCreate], db: Session = Depends(get_db), ): """ - Create one or a list of new modifiers. - - Returns the created modifier or list of modifiers. + Create a list of new modifiers. """ - return await CRUD_modifier.create( - db=db, obj_in=modifier, return_nothing=return_nothing - ) + return await CRUD_modifier.create(db=db, modifiers=modifiers) @router.put( "/", - response_model=schemas.Modifier, dependencies=[Depends(get_current_active_superuser)], ) async def update_modifier( - modifierId: int, - position: int, - modifier_update: schemas.ModifierUpdate, + modifier: schemas.ModifierUpdate, db: Session = Depends(get_db), ): """ @@ -153,15 +142,7 @@ async def update_modifier( Returns the updated modifier. """ - - modifier_map = {"modifierId": modifierId, "position": position} - - modifier = await CRUD_modifier.get( - db=db, - filter=modifier_map, - ) - - return await CRUD_modifier.update(db_obj=modifier, obj_in=modifier_update, db=db) + return await CRUD_modifier.update(db=db, modifier=modifier) @router.delete( diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index 21f34b91..e2691c6c 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -2,6 +2,7 @@ from datetime import datetime from sqlalchemy import ( + ARRAY, Boolean, CheckConstraint, DateTime, @@ -15,6 +16,7 @@ SmallInteger, String, Text, + UniqueConstraint, func, ) from sqlalchemy.dialects.postgresql import JSONB, UUID @@ -89,7 +91,7 @@ class ItemAvailability(Base): ) itemId: Mapped[str] = mapped_column( Text, - ForeignKey("item_temp.itemId", ondelete="CASCADE", onupdate="CASCADE"), + ForeignKey("item.itemId", ondelete="CASCADE", onupdate="CASCADE"), nullable=False, ) @@ -105,7 +107,10 @@ class ItemAvailability(Base): isAsync: Mapped[bool | None] = mapped_column(Boolean, nullable=False) - __table_args__ = (Index("ix_item_id_valid_from", "itemId", "validFrom"),) + __table_args__ = ( + Index("ix_item_id_valid_from", "itemId", "validFrom"), + UniqueConstraint("itemId", "validFrom"), + ) class Item(Base): @@ -151,6 +156,9 @@ class Item(Base): "leagueId", "itemBaseTypeId", ), + UniqueConstraint( + "gameItemId", "leagueId", name="uq_item_game_item_id_league_id" + ), ) @@ -218,10 +226,14 @@ class UnidentifiedItem(_ItemBase, Base): class Modifier(Base): __tablename__ = "modifier" - modifierId: Mapped[int] = mapped_column(SmallInteger, nullable=False) - position: Mapped[int] = mapped_column(SmallInteger, nullable=False) - minRoll: Mapped[float | None] = mapped_column(Float(4)) - maxRoll: Mapped[float | None] = mapped_column(Float(4)) + modifierId: Mapped[int] = mapped_column(SmallInteger, primary_key=True) + + static: Mapped[bool | None] = mapped_column(Boolean) + effect: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + regex: Mapped[str | None] = mapped_column(Text) + + relatedUniques: Mapped[str | None] = mapped_column(Text) + implicit: Mapped[bool | None] = mapped_column(Boolean) explicit: Mapped[bool | None] = mapped_column(Boolean) delve: Mapped[bool | None] = mapped_column(Boolean) @@ -231,11 +243,7 @@ class Modifier(Base): corrupted: Mapped[bool | None] = mapped_column(Boolean) enchanted: Mapped[bool | None] = mapped_column(Boolean) veiled: Mapped[bool | None] = mapped_column(Boolean) - static: Mapped[bool | None] = mapped_column(Boolean) - effect: Mapped[str] = mapped_column(Text, nullable=False) - relatedUniques: Mapped[str | None] = mapped_column(Text) - textRolls: Mapped[str | None] = mapped_column(Text) - regex: Mapped[str | None] = mapped_column(Text) + createdAt: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=func.now(), nullable=False ) @@ -245,29 +253,15 @@ class Modifier(Base): ) __table_args__ = ( - PrimaryKeyConstraint("modifierId", "position"), CheckConstraint( """ CASE WHEN (modifier.static = TRUE) THEN ( - (modifier."minRoll" IS NULL AND modifier."maxRoll" IS NULL) - AND modifier."textRolls" IS NULL - AND modifier.regex IS NULL + modifier.regex IS NULL ) ELSE ( - ( - ( - (modifier."minRoll" IS NOT NULL AND modifier."maxRoll" IS NOT NULL) - AND modifier."textRolls" IS NULL - ) - OR - ( - (modifier."minRoll" IS NULL AND modifier."maxRoll" IS NULL) - AND modifier."textRolls" IS NOT NULL - ) - ) - AND modifier.regex IS NOT NULL + modifier.regex IS NOT NULL ) END """, @@ -287,17 +281,33 @@ class Modifier(Base): """, name="check_modifier_if_not_static_then_modifier_contains_hashtag", ), + ) + + +class ModifierRoll(Base): + __tablename__ = "modifier_roll" + + modifierId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey("modifier.modifierId", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + index=True, + ) + position: Mapped[int] = mapped_column(SmallInteger, nullable=False) + minRoll: Mapped[float | None] = mapped_column(Float(4)) + maxRoll: Mapped[float | None] = mapped_column(Float(4)) + textRolls: Mapped[list[str] | None] = mapped_column(ARRAY(Text())) + + __table_args__ = ( + PrimaryKeyConstraint("modifierId", "position"), CheckConstraint( - """ modifier."maxRoll" >= modifier."minRoll" """, + """ modifier_roll."maxRoll" >= modifier_roll."minRoll" """, name="check_modifier_maxRoll_greaterThan_minRoll", ), ) class ItemModifier(Base): - # Hypertable - # For hypertable specs, see alembic revision `cc29b89156db' - __tablename__ = "item_modifier" itemId: Mapped[int] = mapped_column( Integer, @@ -319,7 +329,7 @@ class ItemModifier(Base): PrimaryKeyConstraint("itemId", "modifierId", "position"), ForeignKeyConstraint( ["modifierId", "position"], - ["modifier.modifierId", "modifier.position"], + ["modifier_roll.modifierId", "modifier_roll.position"], ondelete="CASCADE", onupdate="CASCADE", ), diff --git a/src/backend_api/app/core/schemas/__init__.py b/src/backend_api/app/core/schemas/__init__.py index 83ee7ebf..d8536777 100644 --- a/src/backend_api/app/core/schemas/__init__.py +++ b/src/backend_api/app/core/schemas/__init__.py @@ -1,6 +1,7 @@ "All schemas are imported here and then exported to the main file" from .currency import ( + Currency, CurrencyPrice, CurrencyPriceCreate, CurrencyPriceInDB, @@ -9,14 +10,13 @@ CurrencyTypeCreate, CurrencyTypeInDB, CurrencyTypeUpdate, - Currency, ) -from .item import Item, ItemCreate, ItemInDB, ItemUpdate -from .item_availability import ( +from .item import ( + Item, ItemAvailability, - ItemAvailabilityCreate, - ItemAvailabilityInDB, - ItemAvailabilityUpdate, + ItemCreate, + ItemQuery, + ItemUpdate, ) from .item_base_type import ( ItemBaseType, @@ -27,8 +27,6 @@ from .item_modifier import ( ItemModifier, ItemModifierCreate, - ItemModifierInDB, - ItemModifierUpdate, ) from .league import ( League, @@ -38,10 +36,9 @@ ) from .message import Message from .modifier import ( - GroupedModifierByEffect, + GroupedModifier, Modifier, ModifierCreate, - ModifierInDB, ModifierUpdate, ) from .token import NewPassword, Token, TokenPayload diff --git a/src/backend_api/app/core/schemas/item_modifier.py b/src/backend_api/app/core/schemas/item_modifier.py index e04a9ba5..39642e45 100644 --- a/src/backend_api/app/core/schemas/item_modifier.py +++ b/src/backend_api/app/core/schemas/item_modifier.py @@ -2,7 +2,7 @@ # Shared item modifier props -class _BaseItemModifier(_pydantic.BaseModel): +class ItemModifier(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) itemId: int @@ -11,26 +11,12 @@ class _BaseItemModifier(_pydantic.BaseModel): roll: float | None = None -# Properties to receive on item modifier creation -class ItemModifierCreate(_BaseItemModifier): - pass - - -# Properties to receive on update -class ItemModifierUpdate(_BaseItemModifier): - pass - - -# Properties shared by models stored in DB -class ItemModifierInDBBase(_BaseItemModifier): - pass - +class ModifierRoll(_pydantic.BaseModel): + position: int + roll: float | None = None -# Properties to return to client -class ItemModifier(ItemModifierInDBBase): - pass +class ItemModifierCreate(_pydantic.BaseModel): + modifierId: int -# Properties stored in DB -class ItemModifierInDB(ItemModifierInDBBase): - pass + rolls: list[ModifierRoll] | None = None diff --git a/src/backend_api/app/core/schemas/modifier.py b/src/backend_api/app/core/schemas/modifier.py index 75d2d621..a8ff0ee2 100644 --- a/src/backend_api/app/core/schemas/modifier.py +++ b/src/backend_api/app/core/schemas/modifier.py @@ -1,20 +1,18 @@ -import datetime as _dt +import re import pydantic as _pydantic # Shared modifier props -class _BaseModifier(_pydantic.BaseModel): - model_config = _pydantic.ConfigDict(from_attributes=True) +class ModifierBase(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True, extra="ignore") - position: int relatedUniques: str | None = None - minRoll: float | None = None - maxRoll: float | None = None - textRolls: str | None = None + static: bool | None = None effect: str - regex: str | None = None + regex: re.Pattern | None = None + implicit: bool | None = None explicit: bool | None = None delve: bool | None = None @@ -25,43 +23,46 @@ class _BaseModifier(_pydantic.BaseModel): enchanted: bool | None = None veiled: bool | None = None + @_pydantic.field_validator("regex", mode="before") + @classmethod + def compile_regex(cls, value: str | None) -> re.Pattern: + if value is None: + return None + elif isinstance(value, str): + return re.compile(value) -class GroupedModifierProperties(_pydantic.BaseModel): - position: list[int] - textRolls: list[str | None] - + return value -class GroupedModifierByEffect(_pydantic.BaseModel): - modifierId: int - effect: str - regex: str - static: bool | None - relatedUniques: str | None - groupedModifierProperties: GroupedModifierProperties + @_pydantic.field_serializer("regex") + def serialize_regex(self, regex: re.Pattern | None) -> str | None: + if regex is None: + return None + elif isinstance(regex, str): + return regex + else: + return regex.pattern -# Properties to receive on modifier creation -class ModifierCreate(_BaseModifier): - pass +class ModifierRoll(_pydantic.BaseModel): + position: int + minRoll: float | None = None + maxRoll: float | None = None + textRolls: list[str] | None = None -# Properties to receive on update -class ModifierUpdate(_BaseModifier): - pass +# Properties to receive on modifier creation +class ModifierCreate(ModifierBase): + rolls: list[ModifierRoll] = _pydantic.Field(default_factory=list) -# Properties shared by models stored in DB -class ModifierInDBBase(_BaseModifier): +# Properties to return to client +class Modifier(ModifierBase): modifierId: int - createdAt: _dt.datetime - updatedAt: _dt.datetime | None = None -# Properties to return to client -class Modifier(ModifierInDBBase): - pass +class GroupedModifier(Modifier): + rolls: list[ModifierRoll] = _pydantic.Field(default_factory=list) -# Properties stored in DB -class ModifierInDB(ModifierInDBBase): - pass +class ModifierUpdate(GroupedModifier): + effect: str | None = None diff --git a/src/backend_api/app/crud/__init__.py b/src/backend_api/app/crud/__init__.py index 3d6e467f..1c67fe76 100644 --- a/src/backend_api/app/crud/__init__.py +++ b/src/backend_api/app/crud/__init__.py @@ -1,7 +1,6 @@ from app.core.models.models import Item as model_Item from app.core.models.models import ItemAvailability as model_ItemAvailability from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.core.models.models import ItemModifier as model_ItemModifier from app.core.schemas.item import Item, ItemCreate, ItemUpdate from app.core.schemas.item_availability import ( ItemAvailability, @@ -13,12 +12,8 @@ ItemBaseTypeCreate, ItemBaseTypeUpdate, ) -from app.core.schemas.item_modifier import ( - ItemModifier, - ItemModifierCreate, - ItemModifierUpdate, -) from app.crud.extensions.crud_currency import CRUDCurrency +from app.crud.extensions.crud_item import CRUDItem from app.crud.extensions.crud_league import CRUDLeague from app.crud.extensions.crud_modifier import CRUDModifier from app.crud.extensions.crud_unidentifiedItem import CRUDUnidentifiedItem @@ -45,18 +40,6 @@ model_ItemBaseType, ItemBaseType, ItemBaseTypeCreate, ItemBaseTypeUpdate ](model=model_ItemBaseType, schema=ItemBaseType, create_schema=ItemBaseTypeCreate) - -CRUD_itemModifier = CRUDBase[ - model_ItemModifier, - ItemModifier, - ItemModifierCreate, - ItemModifierUpdate, -]( - model=model_ItemModifier, - schema=ItemModifier, - create_schema=ItemModifierCreate, -) - CRUD_item = CRUDBase[ model_Item, Item, diff --git a/src/backend_api/app/crud/extensions/crud_modifier.py b/src/backend_api/app/crud/extensions/crud_modifier.py index a4dfc3e7..2bff5eca 100644 --- a/src/backend_api/app/crud/extensions/crud_modifier.py +++ b/src/backend_api/app/crud/extensions/crud_modifier.py @@ -1,11 +1,12 @@ -from fastapi import HTTPException from pydantic import TypeAdapter -from sqlalchemy import func, select +from sqlalchemy import func, select, update +from sqlalchemy.dialects.postgresql import aggregate_order_by from sqlalchemy.orm import Session from app.core.models.models import Modifier as model_Modifier +from app.core.models.models import ModifierRoll as model_ModifierRoll from app.core.schemas.modifier import ( - GroupedModifierByEffect, + GroupedModifier, Modifier, ModifierCreate, ModifierUpdate, @@ -28,36 +29,77 @@ def __init__(self): create_schema=ModifierCreate, ) - async def get_grouped_modifier_by_effect( - self, db: Session - ) -> GroupedModifierByEffect: - stmt = select( - model_Modifier.modifierId, - func.min(model_Modifier.effect).label("effect"), - func.coalesce( - func.min(model_Modifier.regex), - func.min(model_Modifier.effect), - ).label("regex"), - func.min(model_Modifier.relatedUniques).label("relatedUniques"), - func.bool_or(model_Modifier.static).label("static"), - func.json_build_object( - "position", - func.json_agg(model_Modifier.position), - "textRolls", - func.json_agg(model_Modifier.textRolls), - ).label("groupedModifierProperties"), - ).group_by(model_Modifier.modifierId) - - grouped_modifier_by_effect_record = db.execute(stmt).mappings().all() - - if not grouped_modifier_by_effect_record: - raise HTTPException( - status_code=404, - detail=f"No objects found in the table {self.model.__tablename__}.", + async def create( + self, db: Session, *, modifiers: list[ModifierCreate] + ) -> list[Modifier]: + db_modifiers = [ + model_Modifier(**modifier.model_dump(exclude="rolls")) + for modifier in modifiers + ] + + db.add_all(db_modifiers) + db.flush() + + db_modifier_rolls = [ + model_ModifierRoll( + modifierId=db_modifier.modifierId, **modifier_roll.model_dump() + ) + for db_modifier, modifier in zip(db_modifiers, modifiers, strict=True) + for modifier_roll in modifier.rolls + ] + db.add_all(db_modifier_rolls) + db.flush() + db.commit() + + return self.validate(db_modifiers) + + async def update(self, db: Session, *, modifier: ModifierUpdate): + update_modifier_stmt = ( + update(model_Modifier) + .where(model_Modifier.modifierId == modifier.modifierId) + .values(**modifier.model_dump(exclude_unset=True, exclude_none=True)) + ) + db.execute(update_modifier_stmt) + + modifier_rolls = [ + {"modifierId": modifier.modifierId, **roll.model_dump(exclude_unset=True)} + for roll in modifier.rolls + ] + if modifier_rolls: + db.execute(update(model_ModifierRoll), modifier_rolls) + db.commit() + + async def get_grouped_modifiers(self, db: Session) -> list[GroupedModifier]: + grouped_modifier_rolls = ( + select( + model_ModifierRoll.modifierId, + func.json_agg( + aggregate_order_by( + func.json_build_object( + "position", + model_ModifierRoll.position, + "minRoll", + model_ModifierRoll.minRoll, + "maxRoll", + model_ModifierRoll.maxRoll, + "textRolls", + model_ModifierRoll.textRolls, + ), + model_ModifierRoll.modifierId, + model_ModifierRoll.position, + ), + ).label("rolls"), ) + .group_by(model_ModifierRoll.modifierId) + .cte("grouped_modifier_rolls") + ) - validate = TypeAdapter( - GroupedModifierByEffect | list[GroupedModifierByEffect] - ).validate_python + stmt = select( + *model_Modifier.__table__.columns, grouped_modifier_rolls.c.rolls + ).join( + grouped_modifier_rolls, + model_Modifier.modifierId == grouped_modifier_rolls.c.modifierId, + ) + grouped_modifiers = db.execute(stmt).mappings().all() - return validate(grouped_modifier_by_effect_record) + return TypeAdapter(list[GroupedModifier]).validate_python(grouped_modifiers) diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py index fc1c8cf0..814aa1f4 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py @@ -1,6 +1,5 @@ from collections.abc import Iterator -import requests from backend_api.app.core.schemas.currency import ( CurrencyType, CurrencyTypeCreate, @@ -15,7 +14,7 @@ ) from data_retrieval_app.logs.logger import data_deposit_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import get_data_safe, post_data_safe +from data_retrieval_app.utils import get_data_safe, send_request_safe class CurrencyTypeDataDepositor(DataDepositorBase): @@ -139,18 +138,13 @@ def _process_data( currencyId=currency_id, name=currency.name, tradeName=trade_name ) - try: - response = requests.put( - self.data_url, - json=updated_currency.model_dump(), - headers=headers, - ) - response.raise_for_status() - except Exception as e: - logger.error( - f"The following error occurred while inserting data: {e}" - ) - raise e + send_request_safe( + "put", + self.data_url, + json=updated_currency.model_dump(), + headers=self.pom_auth_headers, + logger=logger, + ) elif not found_duplicate: processed_currencies.append( @@ -172,7 +166,8 @@ def _insert_data(self, currencies: list[CurrencyTypeCreate]): headers = {"accept": "application/json", "Content-Type": "application/json"} headers.update(self.pom_auth_headers) - post_data_safe( + send_request_safe( + "post", self.data_url, json=TypeAdapter(list[CurrencyTypeCreate]).dump_python(currencies), headers=headers, diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py index d9c606af..1713f1b2 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py @@ -1,200 +1,218 @@ -from io import StringIO - -import pandas as pd -import requests +import csv +import os +from collections.abc import Iterator + +from backend_api.app.core.schemas.modifier import ( + GroupedModifier, + ModifierCreate, + ModifierRoll, + ModifierUpdate, +) +from pydantic import TypeAdapter from data_retrieval_app.data_deposit.data_depositor_base import DataDepositorBase from data_retrieval_app.data_deposit.modifier.modifier_processing_modules import ( ModifierRegexCreator, - check_for_additional_modifier_types, - check_for_new_related_unique, - check_for_updated_numerical_rolls, - check_for_updated_text_rolls, - do_update_regex, ) from data_retrieval_app.logs.logger import data_deposit_logger as logger -from data_retrieval_app.utils import df_to_JSON, get_data_safe - -CASCADING_UPDATE = True +from data_retrieval_app.utils import send_request_safe class ModifierDataDepositor(DataDepositorBase): def __init__(self) -> None: super().__init__(data_type="modifier") - - self.modifier_types = [ - "implicit", - "explicit", - "delve", - "fractured", - "synthesised", - "unique", - "corrupted", - "enchanted", - "veiled", - ] self.regex_creator = ModifierRegexCreator() - self.update_disabled = not CASCADING_UPDATE - - def _get_current_modifiers(self) -> pd.DataFrame: - logger.info("Retrieving previously deposited data.") - - response = get_data_safe( - self.data_url, headers=self.pom_auth_headers, logger=logger + def _get_current_modifiers(self) -> dict[str, GroupedModifier]: + response = send_request_safe( + "get", + f"{self.data_url}grouped/", + headers=self.pom_auth_headers, + logger=logger, ) - json_io = StringIO(response.content.decode("utf-8")) - df = pd.read_json(json_io, dtype=str) - - if df.empty: - logger.info("Found no previously deposited data.") - return None - else: - logger.info("Successfully retrieved previously deposited data.") - return df - - def _update_duplicates( - self, duplicate_df: pd.DataFrame, current_modifiers_df: pd.DataFrame - ) -> None: - if self.update_disabled: - return None - logger.info("Checking if duplicates contain updated information.") - - current_duplicate_modifiers_df = current_modifiers_df.loc[ - current_modifiers_df["effect"].isin(duplicate_df["effect"]) - ].copy() - - # We sort them so that they line up. - # We go through in reverse, as we wish to start with the row that has the highest position. - current_duplicate_modifiers_df.sort_values( - by=["effect", "position"], ascending=False, inplace=True - ) - duplicate_df.sort_values( - by=["effect", "position"], ascending=False, inplace=True - ) - - update_url = self.data_url + "?modifierId={}&position={}" + modifiers = TypeAdapter(list[GroupedModifier]).validate_python(response.json()) + + return {modifier.effect: modifier for modifier in modifiers} + + def _check_for_updates( + self, modifier: ModifierCreate, current_modifier: GroupedModifier + ): + need_update = False + updated_modifier = ModifierUpdate(modifierId=current_modifier.modifierId) + for field in modifier.model_fields: + if field == "rolls": + continue + if field == "relatedUniques": + new = modifier.relatedUniques.split("|") + old = current_modifier.relatedUniques.split("|") + related_uniques = set(new).difference(set(old)) + if related_uniques: + updated_modifier.relatedUniques = "|".join(set(new) | set(old)) + need_update = True + + continue + + if field == "regex": + continue + + new = getattr(modifier, field) + old = getattr(current_modifier, field) + if not (new == old or (isinstance(old, bool) and old)): + # update rows which have a different value + # ignore rows which are bools that are already True + setattr(updated_modifier, field, new) + need_update = True - rolls = None update_regex = False - for (_, row_cur), (_, row_new) in zip( - current_duplicate_modifiers_df.iterrows(), - duplicate_df.iterrows(), - strict=False, + for new_roll, old_roll in zip( + modifier.rolls, current_modifier.rolls, strict=True ): - put_update = False - data = df_to_JSON(row_cur, request_method="put") - position = int(data["position"]) - # if position is higher than 1, we want to store the types of rolls it has - if position >= 1 and rolls is None: - update_regex = False - effect = data["effect"] - same_modifier_df = duplicate_df.loc[ - duplicate_df["effect"] == effect - ].copy() - same_modifier_df.sort_values( - by="position", inplace=True - ) # So that the rolls are added in the correct order - rolls = [] - for _, same_modifier_row in same_modifier_df.iterrows(): - if not pd.isna(same_modifier_row["static"]): - pass - elif not pd.isna(same_modifier_row["textRolls"]): - rolls.append(same_modifier_row["textRolls"]) - else: - rolls.append(None) - - if "updatedAt" in data: - data.pop("updatedAt") - - if not pd.isna(row_new["static"]): - pass - elif not pd.isna(row_new["textRolls"]): - data, put_update, rolls = check_for_updated_text_rolls( - data=data, - row_new=row_new, - rolls=rolls, - regex_creator=self.regex_creator, + updated_roll = ModifierRoll(position=new_roll.position) + roll_need_update = False + if new_roll.minRoll is not None and new_roll.minRoll < old_roll.minRoll: + updated_roll.minRoll = new_roll.minRoll + roll_need_update = True + + if new_roll.maxRoll is not None and new_roll.maxRoll > old_roll.maxRoll: + updated_roll.maxRoll = new_roll.maxRoll + roll_need_update = True + + if new_roll.textRolls is not None: + text_rolls = set(new_roll.textRolls).difference(set(old_roll.textRolls)) + if text_rolls: + # preserve order to not disturb existing data + updated_roll.textRolls = old_roll.textRolls + list(text_rolls) + roll_need_update = True + update_regex = True # regex contains text rolls + + if roll_need_update: + updated_modifier.rolls.append(updated_roll) + need_update = True + + if update_regex: + updated_modifier.regex = self.regex_creator.make_regex( + modifier.effect, updated_modifier.rolls + ) + need_update = True + + if need_update: + send_request_safe( + "put", + self.data_url, + json=updated_modifier.model_dump(exclude_none=True, exclude_unset=True), + headers=self.pom_auth_headers, + logger=logger, + ) + + return need_update + + def _remove_duplicates( + self, modifiers: list[ModifierCreate] + ) -> list[ModifierCreate]: + current_modifiers = self._get_current_modifiers() + + previous_effects = list[str]() + did_update = False + for modifier in modifiers[:]: + if modifier.effect in current_modifiers: + did_update = self._check_for_updates( + modifier, current_modifiers[modifier.effect] ) - update_regex = put_update - else: - data, put_update = check_for_updated_numerical_rolls( - data=data, row_new=row_new + modifiers.remove(modifier) + continue + + if modifier.effect in previous_effects: + modifiers.remove(modifier) + continue + + previous_effects.append(modifier.effect) + if did_update: + logger.info("Updated modifiers using new data") + return modifiers + + def _track_comments(self, modifiers: list[ModifierCreate]) -> list[ModifierCreate]: + unique_name = self.logged_file_comments["Unique Name"] + for modifier in modifiers: + modifier.relatedUniques = unique_name + + return modifiers + + def _load_data(self) -> Iterator[ModifierCreate]: + for filename in os.listdir(self.new_data_location): + modifiers = list[dict]() + filepath = os.path.join(self.new_data_location, filename) + + self.logged_file_comments = {} + logger.info(f"Loading new data from '{filename}'.") + with open(filepath) as infile: + while True: + position = infile.tell() + line = infile.readline() + + if not line: + break + + if line.startswith("#"): + logger.info(line.rstrip()) + split_line = line[1:].split(":", 1) + self.logged_file_comments[split_line[0].strip()] = split_line[ + 1 + ].strip() + else: + # We found the CSV header, so go back to its beginning + infile.seek(position) + break + + modifiers.extend(csv.DictReader(infile)) + + deposit_modifiers = dict[str, ModifierCreate]() + for modifier in modifiers: + for key in list(modifier.keys()): + if modifier[key] == "": + modifier.pop(key) + + text_roll: str | None = modifier.get("textRolls") + if text_roll is not None: + text_roll = text_roll.split("|") + + roll = ModifierRoll( + position=modifier["position"], + minRoll=modifier.get("minRoll"), + maxRoll=modifier.get("maxRoll"), + textRolls=text_roll, ) - if update_regex: - data = do_update_regex(data, rolls, regex_creator=self.regex_creator) - put_update = True + deposit_modifier = deposit_modifiers.get(modifier["effect"]) + if deposit_modifier is None: + deposit_modifier = ModifierCreate(**modifier) + deposit_modifiers[modifier["effect"]] = deposit_modifier - data, put_update = check_for_additional_modifier_types( - data=data, - put_update=put_update, - row_new=row_new, - modifier_types=self.modifier_types, - ) + deposit_modifier.rolls.append(roll) - data, put_update = check_for_new_related_unique( - data=data, - put_update=put_update, - new_related_unique=self.logged_file_comments["Unique Name"], + yield TypeAdapter(list[ModifierCreate]).validate_python( + deposit_modifiers.values() ) - if put_update: - logger.info("Pushed updated modifier to the database.") - headers = { - "accept": "application/json", - "Content-Type": "application/json", - } - headers.update(self.pom_auth_headers) - try: - response = requests.put( - update_url.format(row_cur["modifierId"], row_cur["position"]), - json=data, - headers=headers, - # add HTTP Basic Auth - ) - response.raise_for_status() - except Exception as e: - logger.error( - f"The following error occurred while making request during _update_duplicates modifiers: {e}" - ) - raise e - - # We reset the rolls if the position is 0, because then the next row will be a new modifier - if position == 0 and rolls is not None: - rolls = None - update_regex = False - - def _remove_duplicates(self, new_modifiers_df: pd.DataFrame) -> pd.DataFrame: - current_modifiers_df = self._get_current_modifiers() - - new_modifiers_df = new_modifiers_df.drop_duplicates() - - if current_modifiers_df is None: - logger.info("Skipping duplicate removing due to no previous data") - return new_modifiers_df - - logger.info("Removing duplicate modifiers") - duplicate_mask = ( - new_modifiers_df["effect"] - .str.lower() - .isin(current_modifiers_df["effect"].str.lower()) + def _process_data(self, modifiers: list[ModifierCreate]) -> list[ModifierCreate]: + modifiers = self.regex_creator.add_regex(modifiers) + modifiers = self._track_comments(modifiers) + modifiers = self._remove_duplicates(modifiers) + return modifiers + + def _insert_data(self, modifiers: list[ModifierCreate]): + if not modifiers: + return + + logger.info("Inserting data into database.") + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_auth_headers) + + send_request_safe( + "post", + self.data_url, + json=TypeAdapter(list[ModifierCreate]).dump_python(modifiers), + headers=headers, ) - duplicate_df = new_modifiers_df.loc[duplicate_mask].copy() - self._update_duplicates(duplicate_df, current_modifiers_df) - non_duplicate_df = new_modifiers_df.loc[~duplicate_mask].copy() - - return non_duplicate_df - - def _track_comments(self, df: pd.DataFrame) -> pd.DataFrame: - df["relatedUniques"] = self.logged_file_comments["Unique Name"] - - return df - - def _process_data(self, df: pd.DataFrame) -> pd.DataFrame: - df = self.regex_creator.add_regex(df.copy()) - df = self._remove_duplicates(df.copy()) - df = self._track_comments(df.copy()) - return df + logger.info("Successfully inserted data into database.") diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py index 164b80af..12464a18 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py @@ -1,342 +1,24 @@ import re -from typing import Any -import pandas as pd - -from data_retrieval_app.logs.logger import data_deposit_logger as logger +from backend_api.app.core.schemas.modifier import ModifierCreate, ModifierRoll class ModifierRegexCreator: - def __init__(self): - self.modifier_df_required_columns = [ - "minRoll", - "maxRoll", - "textRolls", - "position", - "effect", - "static", - ] - - def _divide_into_dynamic_static( - self, modifier_df: pd.DataFrame - ) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - Divides the modifier dataframe based on the `static` attribute. - """ - logger.debug("Dividing modifier dataframe into dynamic and static modifiers") - - static_modifier_mask = modifier_df["static"] == "True" - - dynamic_modifier_df = ( - modifier_df.loc[~static_modifier_mask].copy().reset_index() - ) - static_modifier_df = modifier_df.loc[static_modifier_mask].copy().reset_index() - - logger.debug("Successfully divided modifier dataframe") - - return dynamic_modifier_df, static_modifier_df - - def _prepare_df(self, dynamic_modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Escapes the `+`, as it is a regex quantifier. - """ - dynamic_modifier_df["effect"] = dynamic_modifier_df["effect"].str.replace( - "+", r"\+" - ) # Do we need to do the same for `-`? Can other regex symbols occur? - - return dynamic_modifier_df - - def _pre_process( - self, modifier_df: pd.DataFrame - ) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - Performs pre-process steps: - 1. The program assumes certain columns are present to work. These - columns are filled with `pd.NA` if not present. - 2a. Divides the modifiers based on their `static` attribute. - 2b. Returns only the static modifier df if no dyamic modifiers - are found. - 3. Prepares the dynamic modifier df for regex steps. - """ - modifier_df = modifier_df.reindex( - columns=modifier_df.columns.union(self.modifier_df_required_columns) - ) - - dynamic_modifier_df, static_modifier_df = self._divide_into_dynamic_static( - modifier_df=modifier_df - ) - if dynamic_modifier_df.empty: - logger.debug("Only static modifiers present") - return None, static_modifier_df - - dynamic_modifier_df = self._prepare_df(dynamic_modifier_df.copy()) - logger.debug("Finished pre-processing steps") - - return dynamic_modifier_df, static_modifier_df - - def _group_df(self, dynamic_modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Groups the dynamic modifier based on effect. - `position` and `textRolls` are combined into lists of - equal length. - """ - logger.debug( - "Preparing dynamic modifier dataframe for regex conversion, by grouping by effect" - ) - agg_dict = { - "position": lambda positions: list(set(positions)), - "textRolls": lambda rolls: [ - roll if not pd.isna(roll) else None for roll in rolls - ], - } - grouped_dynamic_modifier_df = dynamic_modifier_df.groupby( - "effect", as_index=False, sort=False - ).agg(agg_dict) - logger.debug("Successfully grouped modifiers by effect") - - return grouped_dynamic_modifier_df - - def create_regex_from_row(self, row: pd.DataFrame | dict) -> pd.DataFrame: - """ - A method available for both internal and external use. - - Uses the aggregated `textRolls` field to determine wether - the corresponding position has a numerical (textRoll is None), - or a text roll which determines the regex pattern to insert in - place of the `#`. - - Then it accounts for alternative spelling (eg. `-` instead of `+` or - `reduced` instead of `increased`) - """ - effect: str = row["effect"] - text_rolls: list[str] = row["textRolls"] - - for text_roll in text_rolls: - if text_roll is not None: - effect = effect.replace("#", f"({text_roll})", 1) + def make_regex(self, effect: str, rolls: list[ModifierRoll]) -> str: + regex = effect.replace("+", "[+-]") + for roll in rolls: + if roll.textRolls is not None: + effect = effect.replace("#", f"({roll.textRolls})", 1) else: effect = effect.replace("#", r"([0-9]*[.]?[0-9]+)", 1) - regex = effect.replace(r"\+", r"[+-]") regex = re.sub(r"increased|reduced", "(increased|reduced)", regex) return rf"^{regex}$" - def _unnest_df( - self, - dynamic_modifier_df: pd.DataFrame, - grouped_dynamic_modifier_df: pd.DataFrame, - ) -> pd.DataFrame: - """ - Unnests the grouped modfifier dataframe to original shape. - """ - logger.debug("Reverting the grouped dynamic modifier group to original shape") - exploded_grouped_dynamic_modifier_df = grouped_dynamic_modifier_df.explode( - "position", ignore_index=True - ) - - dynamic_modifier_df["regex"] = exploded_grouped_dynamic_modifier_df["regex"] - logger.debug("Reverted to original shape") - - return dynamic_modifier_df - - def _add_regex(self, dynamic_modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Performs the actual steps of adding regex: - 1. Groups the modifiers for easier processing. - 2. Creates the regex by iterating over each row. - 3. Accounts for alternative wording. - 4. Unnests the grouped modfifier dataframe to original shape. - """ - grouped_dynamic_modifier_df = self._group_df(dynamic_modifier_df.copy()) - - logger.debug("Adding regex row by row") - grouped_dynamic_modifier_df["regex"] = grouped_dynamic_modifier_df.apply( - self.create_regex_from_row, axis=1 - ) - - dynamic_modifier_df = self._unnest_df( - dynamic_modifier_df.copy(), grouped_dynamic_modifier_df.copy() - ) - failed_df = dynamic_modifier_df.loc[dynamic_modifier_df["regex"].isna()] - if not failed_df.empty: - logger.critical("Some modifiers did not get a regex:") - logger.critical(failed_df["effect"]) - raise AssertionError("Some modifiers did not get a regex.") - logger.debug("Successfully added regex row by row") - - return dynamic_modifier_df - - def _post_process( - self, dynamic_modifier_df: pd.DataFrame, static_modifier_df: pd.DataFrame - ) -> pd.DataFrame: - """ - Undos temporary changes. - - Combines dynamic and static modifiers to one modifier dataframe again. - """ - dynamic_modifier_df.loc[:, "effect"] = dynamic_modifier_df[ - "effect" - ].str.replace(r"\+", "+") - - def remove_quantifier(ser: pd.Series) -> pd.Series: - contains_q_mask = ser.str.contains("?", regex=False) - ser_contains_q = ser.loc[contains_q_mask].copy() - - if ser_contains_q.empty: - return ser - - ser_split = ser_contains_q.str.split("?") - for index, str_parts in ser_split.items(): - ser.iloc[index] = "".join( - [ - part[:-1] if i < (len(str_parts) - 1) else part - for i, part in enumerate(str_parts) - ] - ) - return ser - - dynamic_modifier_df.loc[:, "effect"] = remove_quantifier( - dynamic_modifier_df.loc[:, "effect"] - ) - - final_df = pd.concat( - (dynamic_modifier_df, static_modifier_df), ignore_index=True - ) - - return final_df - - def add_regex(self, modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Processes the incoming dataframe, making it ready for adding regex. - If no dynamic modifiers are present, it returns early. - Otherwise, regex is added and post processing steps are applied - to convert the dataframe back to original shape. - """ - dynamic_modifier_df, static_modifier_df = self._pre_process(modifier_df.copy()) - if dynamic_modifier_df is None: - return static_modifier_df - - dynamic_modifier_df = self._add_regex(dynamic_modifier_df.copy()) - - logger.debug("Finishing up the regex addition step") - final_df = self._post_process( - dynamic_modifier_df.copy(), static_modifier_df.copy() - ) - logger.debug("Finished adding regex.") - - return final_df - - -def check_for_updated_text_rolls( - data: dict[str, Any], - row_new: pd.DataFrame, - rolls: list[None | str], - regex_creator: ModifierRegexCreator, -) -> tuple[dict[str, Any], bool, list[None | str]]: - if data["textRolls"] != row_new["textRolls"]: - logger.info( - f"Found a modifier with new 'textRolls'. Modifier: {data['effect']}" - ) - new_rolls: list[str] = row_new["textRolls"].split("|") - old_rolls: list[str] = data["textRolls"].split("|") - combined_rolls = old_rolls + [ - roll - for roll in new_rolls - if roll.lower() not in [old_roll.lower() for old_roll in old_rolls] - ] - if rolls is not None: - rolls[int(data["position"])] = "|".join(combined_rolls) - data["textRolls"] = rolls - else: - data["textRolls"] = [data["textRolls"]] - - data["effect"] = data["effect"].replace("+", r"\+") - data["regex"] = regex_creator.create_regex_from_row(data) - data["effect"] = data["effect"].replace(r"\+", "+") - - data["textRolls"] = "|".join(combined_rolls) - - put_update = True - else: - put_update = False - - return data, put_update, rolls - - -def do_update_regex( - data: dict[str, Any], rolls: list[None, str], regex_creator: ModifierRegexCreator -) -> dict[str, Any]: - if "textRolls" in data: - text_rolls = data["textRolls"] - else: - text_rolls = None - - data["textRolls"] = rolls - data["effect"] = data["effect"].replace("+", r"\+") - data["regex"] = regex_creator.create_regex_from_row(data) - data["effect"] = data["effect"].replace(r"\+", "+") - - if text_rolls is not None: - data["textRolls"] = text_rolls - else: - data.pop("textRolls") - - return data - - -def check_for_updated_numerical_rolls( - data: dict[str, Any], row_new: pd.DataFrame -) -> tuple[dict[str, Any], bool]: - min_roll = data["minRoll"] - max_roll = data["maxRoll"] - - new_min_roll = row_new["minRoll"] - new_max_roll = row_new["maxRoll"] - - if float(min_roll) > float(new_min_roll): - logger.debug( - f"Found a modifier with a lower 'minRoll'. Modifier: {data['effect']}" - ) - data["minRoll"] = new_min_roll - - if float(max_roll) < float(new_max_roll): - logger.info( - f"Found a modifier with a higher 'maxRoll'. Modifier: {data['effect']}" - ) - data["maxRoll"] = new_max_roll - - if min_roll != new_min_roll or max_roll != new_max_roll: - logger.debug("Updating modifier to bring numerical roll range up-to-date.") - put_update = True - else: - put_update = False - - return data, put_update - - -def check_for_additional_modifier_types( - data: dict[str, Any], - row_new: pd.Series, - put_update: bool, - modifier_types: list[str], -) -> tuple[dict[str, Any], bool]: - for modifier_type in modifier_types: - if modifier_type in row_new.index and modifier_type not in data: - logger.info( - f"Added a modifier type to a modifier. Modifier: {data['effect']}" - ) - data[modifier_type] = row_new[modifier_type] - put_update = True - - return data, put_update - - -def check_for_new_related_unique( - data: dict[str, Any], put_update: bool, new_related_unique: str -) -> tuple[dict[str, Any], bool]: - if new_related_unique not in data["relatedUniques"]: - data["relatedUniques"] += "|" + new_related_unique - put_update = True + def add_regex(self, modifiers: list[ModifierCreate]) -> list[ModifierCreate]: + for modifier in modifiers: + if not modifier.static: + modifier.regex = self.make_regex(modifier.effect, modifier.rolls) - return data, put_update + return modifiers diff --git a/src/backend_data_retrieval/data_retrieval_app/utils.py b/src/backend_data_retrieval/data_retrieval_app/utils.py index a5063178..804ee8dc 100644 --- a/src/backend_data_retrieval/data_retrieval_app/utils.py +++ b/src/backend_data_retrieval/data_retrieval_app/utils.py @@ -1,7 +1,7 @@ import logging from collections.abc import Generator from datetime import UTC, datetime -from typing import Any +from typing import Any, Literal import pandas as pd import requests @@ -148,21 +148,35 @@ def get_data_safe( return response -def post_data_safe( +def send_request_safe( + method: Literal["get", "put", "patch", "post", "delete"], url: str, - *, - json: dict, + *args, logger: logging.Logger = None, - params: dict | list = None, - headers: dict[str, str] | None = None, + **kwargs, ) -> requests.Response: + method_func = None + if method == "get": + method_func = requests.get + elif method == "put": + method_func = requests.put + elif method == "patch": + method_func = requests.patch + elif method == "post": + method_func = requests.post + elif method == "delete": + method_func = requests.delete + else: + raise ValueError( + f"Invalid method ({method}), must be one of: get, put, patch, post, delete" + ) try: - response = requests.post(url, json=json, params=params, headers=headers) + response = method_func(url, *args, **kwargs) response.raise_for_status() except Exception as e: if logger is not None: logger.error( - f"The following error occurred while making request a post request to {url}: {e}" + f"The following error occurred while making request a {method} request to {url}: {e}" ) raise e From 45810961ddfb56e45eb254353b8fb3ab4fc22dd2 Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 26 Sep 2026 17:53:29 -0230 Subject: [PATCH 08/16] #830 Added backend_api back to the internal backend api docker container path --- src/backend_api/Dockerfile | 2 +- src/backend_api/alembic.ini | 2 +- src/backend_api/app/alembic/env.py | 4 +- ...c96438_removing_modifier_auto_increment.py | 2 +- .../23d7asdb13df_split_modifier_table.py | 2 +- .../965e766db0a0_added_league_table.py | 2 +- ...9f3f_added_unidentified_aggregation_job.py | 2 +- src/backend_api/app/api/api.py | 5 +- src/backend_api/app/api/api_message_util.py | 7 +-- src/backend_api/app/api/deps.py | 23 ++++--- src/backend_api/app/api/routes/__init__.py | 22 +++---- src/backend_api/app/api/routes/currency.py | 20 +++--- src/backend_api/app/api/routes/item.py | 19 +++--- .../app/api/routes/item_availability.py | 13 ++-- .../app/api/routes/item_base_type.py | 22 +++---- src/backend_api/app/api/routes/league.py | 18 +++--- src/backend_api/app/api/routes/login.py | 21 +++---- src/backend_api/app/api/routes/modifier.py | 22 +++---- src/backend_api/app/api/routes/plot.py | 17 +++-- src/backend_api/app/api/routes/test.py | 27 ++++---- src/backend_api/app/api/routes/turnstile.py | 14 ++--- .../app/api/routes/unidentified_item.py | 18 +++--- src/backend_api/app/api/routes/user.py | 31 +++++----- src/backend_api/app/backend_pre_start.py | 5 +- src/backend_api/app/core/cache/cache.py | 3 +- src/backend_api/app/core/cache/user_cache.py | 11 ++-- src/backend_api/app/core/models/database.py | 3 +- src/backend_api/app/core/models/init_db.py | 9 ++- src/backend_api/app/core/models/models.py | 3 +- .../core/rate_limit/custom_rate_limiter.py | 5 +- .../app/core/rate_limit/rate_limiters.py | 11 ++-- .../app/core/schemas/item_availability.py | 40 ------------ .../app/core/schemas/plot/input.py | 3 +- src/backend_api/app/core/schemas/user.py | 5 +- src/backend_api/app/crud/__init__.py | 42 +++---------- src/backend_api/app/crud/base.py | 19 +++--- .../app/crud/extensions/crud_currency.py | 19 +++--- .../app/crud/extensions/crud_league.py | 13 ++-- .../app/crud/extensions/crud_modifier.py | 17 +++-- .../crud/extensions/crud_unidentifiedItem.py | 15 ++--- src/backend_api/app/crud/user.py | 17 +++-- src/backend_api/app/exception_handlers.py | 7 ++- .../model_exceptions/cache_exception.py | 3 +- .../model_exceptions/db_exception.py | 7 +-- .../model_exceptions/plot_exception.py | 5 +- .../model_exceptions/rate_limit_exception.py | 3 +- .../model_exceptions/request_exception.py | 3 +- .../model_exceptions/test_exception.py | 5 +- .../model_exceptions/user_login_exception.py | 3 +- .../model_exceptions/util_exception.py | 5 +- src/backend_api/app/initial_data.py | 5 +- src/backend_api/app/logs/config/__init__.py | 7 ++- src/backend_api/app/main.py | 29 ++++----- .../app/middleware/request_logs.py | 7 +-- src/backend_api/app/plotting/plotter.py | 62 ++++++++++--------- .../tests/test_real_env/api/real_env_base.py | 11 ++-- .../test_real_env/api/real_env_cases/plot.py | 4 +- .../api/api_routes_test_base.py | 27 ++++---- .../api/api_routes_test_slowapi_rate_limit.py | 9 +-- .../api/api_test_rate_limit_base.py | 4 +- .../api/routes/test_currency.py | 26 ++++---- .../api/routes/test_item.py | 27 ++++---- .../api/routes/test_item_base_type.py | 29 +++++---- .../api/routes/test_item_modifier.py | 36 ++++++----- .../api/routes/test_league.py | 23 ++++--- .../api/routes/test_login.py | 17 ++--- .../api/routes/test_modifier.py | 29 +++++---- .../api/routes/test_plot.py | 17 ++--- .../api/routes/test_turnstile.py | 13 ++-- .../tests/test_simulating_env/base_test.py | 11 ++-- .../app/tests/test_simulating_env/conftest.py | 35 +++++------ .../crud/crud_models/test_currency.py | 8 +-- .../crud/crud_models/test_item.py | 12 ++-- .../crud/crud_models/test_item_base_type.py | 10 +-- .../crud/crud_models/test_item_modifier.py | 29 +++++---- .../crud/crud_models/test_league.py | 8 +-- .../crud/crud_models/test_modifier.py | 6 +- .../crud/crud_models/test_user.py | 2 +- .../crud/crud_test_base.py | 11 ++-- .../crud/crud_test_user.py | 11 ++-- .../setup_test_database.py | 3 +- .../app/tests/utils/database_utils.py | 7 +-- .../app/tests/utils/model_utils/__init__.py | 10 +-- .../app/tests/utils/model_utils/currency.py | 14 ++--- .../app/tests/utils/model_utils/item.py | 20 +++--- .../tests/utils/model_utils/item_base_type.py | 6 +- .../tests/utils/model_utils/item_modifier.py | 16 ++--- .../app/tests/utils/model_utils/league.py | 12 ++-- .../app/tests/utils/model_utils/modifier.py | 12 ++-- .../app/tests/utils/model_utils/plot.py | 9 ++- src/backend_api/app/tests/utils/user.py | 9 ++- src/backend_api/app/tests/utils/utils.py | 5 +- src/backend_api/app/utils/timing_tracker.py | 2 +- src/backend_api/app/utils/user.py | 5 +- .../app/validation/turnstile_validation.py | 9 +-- src/backend_api/scripts/prestart.sh | 6 +- 96 files changed, 601 insertions(+), 665 deletions(-) delete mode 100644 src/backend_api/app/core/schemas/item_availability.py diff --git a/src/backend_api/Dockerfile b/src/backend_api/Dockerfile index 6527843d..34373b5f 100644 --- a/src/backend_api/Dockerfile +++ b/src/backend_api/Dockerfile @@ -33,7 +33,7 @@ COPY ./scripts /app/scripts COPY ./pyproject.toml ./uv.lock ./alembic.ini /app/ -COPY ./app /app/app +COPY ./app /app/backend_api/app # Sync the project # Ref: https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers diff --git a/src/backend_api/alembic.ini b/src/backend_api/alembic.ini index 24841c2b..93864b35 100644 --- a/src/backend_api/alembic.ini +++ b/src/backend_api/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = app/alembic +script_location = backend_api/app/alembic # template used to generate migration files # file_template = %%(rev)s_%%(slug)s diff --git a/src/backend_api/app/alembic/env.py b/src/backend_api/app/alembic/env.py index fff12f53..9a514cfc 100644 --- a/src/backend_api/app/alembic/env.py +++ b/src/backend_api/app/alembic/env.py @@ -6,8 +6,8 @@ import os from alembic import context -from app.core.models.models import Base -from app.core.config import settings +from backend_api.app.core.models.models import Base +from backend_api.app.core.config import settings def get_url(): diff --git a/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py b/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py index 8fc6246d..18a82acf 100644 --- a/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py +++ b/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "17daa1c96438" diff --git a/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py index 1895989d..33133594 100644 --- a/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py +++ b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "23d7asdb13df" diff --git a/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py b/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py index 7586677d..49a9125f 100644 --- a/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py +++ b/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "965e766db0a0" diff --git a/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py b/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py index 47942eee..2f13c596 100644 --- a/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py +++ b/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "e38727349f3f" diff --git a/src/backend_api/app/api/api.py b/src/backend_api/app/api/api.py index 81236ed1..3d4f6b8b 100644 --- a/src/backend_api/app/api/api.py +++ b/src/backend_api/app/api/api.py @@ -1,6 +1,4 @@ -from fastapi import APIRouter - -from app.api.routes import ( +from backend_api.app.api.routes import ( currency, currency_prefix, item, @@ -22,6 +20,7 @@ unidentified_item, unidentified_item_prefix, ) +from fastapi import APIRouter api_router = APIRouter() diff --git a/src/backend_api/app/api/api_message_util.py b/src/backend_api/app/api/api_message_util.py index d4fedd53..c546c595 100644 --- a/src/backend_api/app/api/api_message_util.py +++ b/src/backend_api/app/api/api_message_util.py @@ -1,9 +1,8 @@ -from pydantic import EmailStr - -from app.core.schemas.message import Message -from app.core.schemas.user import ( +from backend_api.app.core.schemas.message import Message +from backend_api.app.core.schemas.user import ( UsernameStr, ) +from pydantic import EmailStr def get_delete_return_msg( diff --git a/src/backend_api/app/api/deps.py b/src/backend_api/app/api/deps.py index e6b51a6a..4c42f0fb 100644 --- a/src/backend_api/app/api/deps.py +++ b/src/backend_api/app/api/deps.py @@ -1,23 +1,22 @@ from collections.abc import AsyncGenerator, Generator from typing import Annotated -from fastapi import Depends, Request -from fastapi.security import OAuth2PasswordBearer -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import Session -from starlette.datastructures import Headers - -from app.core.cache.user_cache import UserCache, UserCacheTokenType -from app.core.config import settings -from app.core.models.database import AsyncSessionLocal, SessionLocal -from app.core.models.models import User -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.exceptions import ( +from backend_api.app.core.cache.user_cache import UserCache, UserCacheTokenType +from backend_api.app.core.config import settings +from backend_api.app.core.models.database import AsyncSessionLocal, SessionLocal +from backend_api.app.core.models.models import User +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.exceptions import ( DbObjectDoesNotExistError, InvalidHeaderProvidedError, UserIsNotActiveError, UserWithNotEnoughPrivilegesError, ) +from fastapi import Depends, Request +from fastapi.security import OAuth2PasswordBearer +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Session +from starlette.datastructures import Headers reusable_oauth2 = OAuth2PasswordBearer( tokenUrl=f"{settings.API_V1_STR}/login/access-token", diff --git a/src/backend_api/app/api/routes/__init__.py b/src/backend_api/app/api/routes/__init__.py index ea1e0336..8887a70a 100644 --- a/src/backend_api/app/api/routes/__init__.py +++ b/src/backend_api/app/api/routes/__init__.py @@ -1,11 +1,11 @@ -from app.api.routes.currency import currency_prefix -from app.api.routes.item import item_prefix -from app.api.routes.item_base_type import item_base_type_prefix -from app.api.routes.league import league_prefix -from app.api.routes.login import login_prefix -from app.api.routes.modifier import modifier_prefix -from app.api.routes.plot import plot_prefix -from app.api.routes.test import test_prefix -from app.api.routes.turnstile import turnstile_prefix -from app.api.routes.unidentified_item import unidentified_item_prefix -from app.api.routes.user import user_prefix +from backend_api.app.api.routes.currency import currency_prefix +from backend_api.app.api.routes.item import item_prefix +from backend_api.app.api.routes.item_base_type import item_base_type_prefix +from backend_api.app.api.routes.league import league_prefix +from backend_api.app.api.routes.login import login_prefix +from backend_api.app.api.routes.modifier import modifier_prefix +from backend_api.app.api.routes.plot import plot_prefix +from backend_api.app.api.routes.test import test_prefix +from backend_api.app.api.routes.turnstile import turnstile_prefix +from backend_api.app.api.routes.unidentified_item import unidentified_item_prefix +from backend_api.app.api.routes.user import user_prefix diff --git a/src/backend_api/app/api/routes/currency.py b/src/backend_api/app/api/routes/currency.py index 6ee0e962..62a5a74e 100644 --- a/src/backend_api/app/api/routes/currency.py +++ b/src/backend_api/app/api/routes/currency.py @@ -1,21 +1,21 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_user_rate_limits, ) -from app.core.schemas.currency import CurrencyQuery -from app.crud import CRUD_currency +from backend_api.app.core.schemas.currency import CurrencyQuery +from backend_api.app.crud import CRUD_currency +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() diff --git a/src/backend_api/app/api/routes/item.py b/src/backend_api/app/api/routes/item.py index cef6074a..5c2f121d 100644 --- a/src/backend_api/app/api/routes/item.py +++ b/src/backend_api/app/api/routes/item.py @@ -1,19 +1,18 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy import text -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( +import backend_api.app.core.schemas as schemas +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_user_rate_limits -from app.crud import CRUD_item +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_user_rate_limits +from backend_api.app.crud import CRUD_item +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy import text +from sqlalchemy.orm import Session router = APIRouter() diff --git a/src/backend_api/app/api/routes/item_availability.py b/src/backend_api/app/api/routes/item_availability.py index 2a0eb33d..723f34e1 100644 --- a/src/backend_api/app/api/routes/item_availability.py +++ b/src/backend_api/app/api/routes/item_availability.py @@ -1,15 +1,14 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( +import backend_api.app.core.schemas as schemas +from backend_api.app.api.deps import ( get_current_active_superuser, get_db, ) -from app.api.params import FilterParams -from app.crud import CRUD_itemAvailability +from backend_api.app.api.params import FilterParams +from backend_api.app.crud import CRUD_itemAvailability +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session router = APIRouter() diff --git a/src/backend_api/app/api/routes/item_base_type.py b/src/backend_api/app/api/routes/item_base_type.py index 1b7a6459..ba671ca5 100644 --- a/src/backend_api/app/api/routes/item_base_type.py +++ b/src/backend_api/app/api/routes/item_base_type.py @@ -1,25 +1,25 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_delete_return_msg, ) -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.models.models import ItemBaseType -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.models.models import ItemBaseType +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.crud import CRUD_itemBaseType +from backend_api.app.crud import CRUD_itemBaseType +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() diff --git a/src/backend_api/app/api/routes/league.py b/src/backend_api/app/api/routes/league.py index af8877bf..2f7b234a 100644 --- a/src/backend_api/app/api/routes/league.py +++ b/src/backend_api/app/api/routes/league.py @@ -1,21 +1,21 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.crud import CRUD_league +from backend_api.app.crud import CRUD_league +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() diff --git a/src/backend_api/app/api/routes/login.py b/src/backend_api/app/api/routes/login.py index 9a666e69..f961dc4a 100644 --- a/src/backend_api/app/api/routes/login.py +++ b/src/backend_api/app/api/routes/login.py @@ -1,24 +1,23 @@ # From FastAPI Fullstack Template https://github.com/fastapi/full-stack-fastapi-template/blob/master/backend/app/api/routes/login.py from typing import Annotated -from fastapi import APIRouter, Depends, Request, Response -from fastapi.security import OAuth2PasswordRequestForm -from sqlalchemy.orm import Session - -from app.api.deps import ( +from backend_api.app.api.deps import ( UserCacheSession, get_db, ) -from app.core.config import settings -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.core.config import settings +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, ) -from app.core.schemas import Token -from app.crud import CRUD_user -from app.exceptions import ( +from backend_api.app.core.schemas import Token +from backend_api.app.crud import CRUD_user +from backend_api.app.exceptions import ( BadLoginCredentialsError, ) +from fastapi import APIRouter, Depends, Request, Response +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy.orm import Session router = APIRouter() diff --git a/src/backend_api/app/api/routes/modifier.py b/src/backend_api/app/api/routes/modifier.py index 4d62967c..81ea9471 100644 --- a/src/backend_api/app/api/routes/modifier.py +++ b/src/backend_api/app/api/routes/modifier.py @@ -1,25 +1,25 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_delete_return_msg, ) -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.models.models import Modifier -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.models.models import Modifier +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.crud import CRUD_modifier +from backend_api.app.crud import CRUD_modifier +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() diff --git a/src/backend_api/app/api/routes/plot.py b/src/backend_api/app/api/routes/plot.py index 4ce04d08..d5972f9d 100644 --- a/src/backend_api/app/api/routes/plot.py +++ b/src/backend_api/app/api/routes/plot.py @@ -1,15 +1,14 @@ -from fastapi import APIRouter, Depends, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import ( +from backend_api.app.api.deps import ( get_async_db, get_user_ip_from_header, ) -from app.core.rate_limit.custom_rate_limiter import RateSpec -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_custom_rate_limit -from app.core.schemas.plot import PlotData, PlotQuery -from app.plotting import configure_plotter_by_query, plotter_service +from backend_api.app.core.rate_limit.custom_rate_limiter import RateSpec +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_custom_rate_limit +from backend_api.app.core.schemas.plot import PlotData, PlotQuery +from backend_api.app.plotting import configure_plotter_by_query, plotter_service +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter() diff --git a/src/backend_api/app/api/routes/test.py b/src/backend_api/app/api/routes/test.py index 85548b95..568a36a1 100644 --- a/src/backend_api/app/api/routes/test.py +++ b/src/backend_api/app/api/routes/test.py @@ -1,22 +1,23 @@ from uuid import uuid4 +from backend_api.app.api.deps import ( + get_current_active_superuser, + get_db, +) +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Item as model_Item +from backend_api.app.core.models.models import User as model_User +from backend_api.app.core.schemas import ItemCreate, User +from backend_api.app.crud import CRUD_user +from backend_api.app.exceptions.model_exceptions.test_exception import ( + OnlyAvailableInLocalEnvError, +) +from backend_api.app.logs.logger import test_logger as logger from fastapi import APIRouter, Depends, HTTPException from httpx import ASGITransport, AsyncClient from sqlalchemy import insert from sqlalchemy.orm import Session -from app.api.deps import ( - get_current_active_superuser, - get_db, -) -from app.core.config import settings -from app.core.models.models import Item as model_Item -from app.core.models.models import User as model_User -from app.core.schemas import ItemCreate, User -from app.crud import CRUD_user -from app.exceptions.model_exceptions.test_exception import OnlyAvailableInLocalEnvError -from app.logs.logger import test_logger as logger - router = APIRouter() @@ -133,7 +134,7 @@ async def bulk_insert_users_and_verify(count: int, db: Session = Depends(get_db) logger.debug("Verifying users") # Verify users by making an internal request to the login route - from app.main import app + from backend_api.app.main import app tokens = [] async with AsyncClient( diff --git a/src/backend_api/app/api/routes/turnstile.py b/src/backend_api/app/api/routes/turnstile.py index 9d160f91..1f448905 100644 --- a/src/backend_api/app/api/routes/turnstile.py +++ b/src/backend_api/app/api/routes/turnstile.py @@ -1,13 +1,13 @@ +from backend_api.app.api.deps import ( + get_user_ip_from_header, +) +from backend_api.app.core.rate_limit.custom_rate_limiter import RateSpec +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_custom_rate_limit +from backend_api.app.validation import turnstile_validation_tool from fastapi import APIRouter, Request import app.core.schemas as schemas -from app.api.deps import ( - get_user_ip_from_header, -) -from app.core.rate_limit.custom_rate_limiter import RateSpec -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_custom_rate_limit -from app.validation import turnstile_validation_tool router = APIRouter() diff --git a/src/backend_api/app/api/routes/unidentified_item.py b/src/backend_api/app/api/routes/unidentified_item.py index cc55c311..33a9d057 100644 --- a/src/backend_api/app/api/routes/unidentified_item.py +++ b/src/backend_api/app/api/routes/unidentified_item.py @@ -1,19 +1,19 @@ from typing import Annotated +from backend_api.app.api.deps import ( + get_current_active_superuser, + get_current_active_user, + get_db, +) +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_user_rate_limits +from backend_api.app.crud import CRUD_unidentifiedItem from fastapi import APIRouter, Depends, Query, Request, Response from sqlalchemy import text from sqlalchemy.orm import Session import app.core.schemas as schemas -from app.api.deps import ( - get_current_active_superuser, - get_current_active_user, - get_db, -) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_user_rate_limits -from app.crud import CRUD_unidentifiedItem router = APIRouter() diff --git a/src/backend_api/app/api/routes/user.py b/src/backend_api/app/api/routes/user.py index a0e099f3..224df60e 100644 --- a/src/backend_api/app/api/routes/user.py +++ b/src/backend_api/app/api/routes/user.py @@ -1,10 +1,7 @@ import uuid from typing import Any -from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response -from sqlalchemy.orm import Session - -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_activation_token_confirmation_sent_msg, get_delete_return_msg, get_set_rate_limit_tier_success_msg, @@ -15,7 +12,7 @@ get_user_update_me_confirmation_sent_msg, get_user_update_me_success_msg, ) -from app.api.deps import ( +from backend_api.app.api.deps import ( CurrentUser, CurrentUserNotActive, UserCacheRegisterSession, @@ -24,14 +21,14 @@ get_current_active_user, get_db, ) -from app.core.config import settings -from app.core.models.models import User -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import User +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.core.schemas import ( +from backend_api.app.core.schemas import ( Message, UpdatePassword, UserCreate, @@ -40,10 +37,10 @@ UsersPublic, UserUpdate, ) -from app.core.schemas.token import Token -from app.core.schemas.user import UserUpdateMe -from app.crud import CRUD_user -from app.exceptions import ( +from backend_api.app.core.schemas.token import Token +from backend_api.app.core.schemas.user import UserUpdateMe +from backend_api.app.crud import CRUD_user +from backend_api.app.exceptions import ( DbObjectAlreadyExistsError, DbObjectDoesNotExistError, InvalidTokenError, @@ -51,13 +48,13 @@ SuperUserNotAllowedToDeleteSelfError, UserWithNotEnoughPrivilegesError, ) -from app.exceptions.model_exceptions.user_login_exception import ( +from backend_api.app.exceptions.model_exceptions.user_login_exception import ( UpdateExisitingMeValuesError, UserEmailRequiredError, UserIsAlreadyActiveError, UserUsernameRequiredError, ) -from app.utils.user import ( +from backend_api.app.utils.user import ( generate_email_changed_notify_email, generate_new_account_email, generate_password_changed_notify_email, @@ -65,6 +62,8 @@ generate_user_registration_email, send_email, ) +from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response +from sqlalchemy.orm import Session router = APIRouter() diff --git a/src/backend_api/app/backend_pre_start.py b/src/backend_api/app/backend_pre_start.py index fbc48689..fca12571 100644 --- a/src/backend_api/app/backend_pre_start.py +++ b/src/backend_api/app/backend_pre_start.py @@ -1,11 +1,10 @@ import logging +from backend_api.app.core.models.database import SessionLocal +from backend_api.app.logs.logger import logger from sqlalchemy import select from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed -from app.core.models.database import SessionLocal -from app.logs.logger import logger - max_tries = 60 * 5 # 5 minutes wait_seconds = 1 diff --git a/src/backend_api/app/core/cache/cache.py b/src/backend_api/app/core/cache/cache.py index 6c6e96c4..9faca654 100644 --- a/src/backend_api/app/core/cache/cache.py +++ b/src/backend_api/app/core/cache/cache.py @@ -1,5 +1,4 @@ import redis.asyncio as Redis - -from app.core.config import settings +from backend_api.app.core.config import settings cache = Redis.from_url(str(settings.CACHE_URI), decode_responses=True) diff --git a/src/backend_api/app/core/cache/user_cache.py b/src/backend_api/app/core/cache/user_cache.py index a35e9742..960e021c 100644 --- a/src/backend_api/app/core/cache/user_cache.py +++ b/src/backend_api/app/core/cache/user_cache.py @@ -2,15 +2,14 @@ from typing import Any from uuid import UUID, uuid4 -from pydantic import TypeAdapter - -from app.core.cache.cache import cache -from app.core.models.models import User as model_User -from app.core.schemas.user import UserInCache -from app.exceptions import ( +from backend_api.app.core.cache.cache import cache +from backend_api.app.core.models.models import User as model_User +from backend_api.app.core.schemas.user import UserInCache +from backend_api.app.exceptions import ( InvalidCacheUpdateParamsError, InvalidTokenError, ) +from pydantic import TypeAdapter class UserCacheTokenType(StrEnum): diff --git a/src/backend_api/app/core/models/database.py b/src/backend_api/app/core/models/database.py index 763fb17a..811759c9 100644 --- a/src/backend_api/app/core/models/database.py +++ b/src/backend_api/app/core/models/database.py @@ -1,10 +1,9 @@ +from backend_api.app.core.config import settings from sqlalchemy import create_engine from sqlalchemy.ext.asyncio import AsyncAttrs, async_sessionmaker, create_async_engine from sqlalchemy.inspection import inspect from sqlalchemy.orm import DeclarativeBase, sessionmaker -from app.core.config import settings - engine = create_engine(str(settings.DATABASE_URI)) async_engine = create_async_engine( # This engine is used for plotting queries str(settings.ASYNC_DATABASE_URI) diff --git a/src/backend_api/app/core/models/init_db.py b/src/backend_api/app/core/models/init_db.py index d2e37d9d..ec61aeb8 100644 --- a/src/backend_api/app/core/models/init_db.py +++ b/src/backend_api/app/core/models/init_db.py @@ -1,11 +1,10 @@ +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import User +from backend_api.app.core.schemas.user import UserCreate +from backend_api.app.crud import CRUD_user from sqlalchemy import select from sqlalchemy.orm import Session -from app.core.config import settings -from app.core.models.models import User -from app.core.schemas.user import UserCreate -from app.crud import CRUD_user - def init_db(session: Session) -> None: user = session.execute( diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index e2691c6c..d89c610a 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -1,6 +1,7 @@ import uuid from datetime import datetime +from backend_api.app.core.models.database import Base from sqlalchemy import ( ARRAY, Boolean, @@ -22,8 +23,6 @@ from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column -from app.core.models.database import Base - class League(Base): __tablename__ = "league" diff --git a/src/backend_api/app/core/rate_limit/custom_rate_limiter.py b/src/backend_api/app/core/rate_limit/custom_rate_limiter.py index 9aaa2a38..77f73fed 100644 --- a/src/backend_api/app/core/rate_limit/custom_rate_limiter.py +++ b/src/backend_api/app/core/rate_limit/custom_rate_limiter.py @@ -9,10 +9,11 @@ AnyRedis, pipeline_expire, ) +from backend_api.app.exceptions.model_exceptions.rate_limit_exception import ( + RateLimitExceededError, +) from typing_extensions import final -from app.exceptions.model_exceptions.rate_limit_exception import RateLimitExceededError - #: These aliases makes our code more readable. _Seconds: TypeAlias = int diff --git a/src/backend_api/app/core/rate_limit/rate_limiters.py b/src/backend_api/app/core/rate_limit/rate_limiters.py index 1325e7bd..a65381c0 100644 --- a/src/backend_api/app/core/rate_limit/rate_limiters.py +++ b/src/backend_api/app/core/rate_limit/rate_limiters.py @@ -2,14 +2,13 @@ from contextlib import asynccontextmanager from typing import Any +from backend_api.app.api.deps import get_user_ip_from_header, get_username_by_request +from backend_api.app.core.cache.cache import cache +from backend_api.app.core.config import settings +from backend_api.app.core.rate_limit.custom_rate_limiter import RateLimiter, RateSpec +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings from slowapi import Limiter -from app.api.deps import get_username_by_request, get_user_ip_from_header -from app.core.cache.cache import cache -from app.core.config import settings -from app.core.rate_limit.custom_rate_limiter import RateLimiter, RateSpec -from app.core.rate_limit.rate_limit_config import rate_limit_settings - def default_limit_provider() -> list[str]: return [ diff --git a/src/backend_api/app/core/schemas/item_availability.py b/src/backend_api/app/core/schemas/item_availability.py deleted file mode 100644 index db323e3c..00000000 --- a/src/backend_api/app/core/schemas/item_availability.py +++ /dev/null @@ -1,40 +0,0 @@ -import pydantic as _pydantic - - -# Shared item props -class _BaseItemAvailability(_pydantic.BaseModel): - model_config = _pydantic.ConfigDict(from_attributes=True) - itemId: int - - currencyId: int - currencyAmount: float - - validFrom: int - validTo: int | None = None - - isAsync: bool | None = None - - -# Properties to receive on item creation -class ItemAvailabilityCreate(_BaseItemAvailability): - pass - - -# Properties to receive on update -class ItemAvailabilityUpdate(_BaseItemAvailability): - pass - - -# Properties shared by models stored in DB -class ItemAvailabilityInDBBase(_BaseItemAvailability): - availabilityId: int - - -# Properties to return to client -class ItemAvailability(ItemAvailabilityInDBBase): - pass - - -# Properties stored in DB -class ItemAvailabilityInDB(ItemAvailabilityInDBBase): - pass diff --git a/src/backend_api/app/core/schemas/plot/input.py b/src/backend_api/app/core/schemas/plot/input.py index 9071528c..05171efd 100644 --- a/src/backend_api/app/core/schemas/plot/input.py +++ b/src/backend_api/app/core/schemas/plot/input.py @@ -1,6 +1,5 @@ import pydantic as _pydantic - -from app.core.schemas.item import Influences +from backend_api.app.core.schemas.item import Influences class ItemSpecs(_pydantic.BaseModel): diff --git a/src/backend_api/app/core/schemas/user.py b/src/backend_api/app/core/schemas/user.py index d31b0d93..adbf7af8 100644 --- a/src/backend_api/app/core/schemas/user.py +++ b/src/backend_api/app/core/schemas/user.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import Annotated +from backend_api.app.core.schemas.wrap_validator import custom_error_msg from pydantic import ( BaseModel, ConfigDict, @@ -10,8 +11,6 @@ StringConstraints, ) -from app.core.schemas.wrap_validator import custom_error_msg - class UsernameValidationError(ValueError): @classmethod @@ -21,7 +20,7 @@ def from_validator_exc( exc: Exception | None, ) -> Exception: return cls( - f"Username must be less than 30 characters and can not contain special symbols or white/empty space.", + "Username must be less than 30 characters and can not contain special symbols or white/empty space.", ) diff --git a/src/backend_api/app/crud/__init__.py b/src/backend_api/app/crud/__init__.py index 1c67fe76..09934fc1 100644 --- a/src/backend_api/app/crud/__init__.py +++ b/src/backend_api/app/crud/__init__.py @@ -1,23 +1,15 @@ -from app.core.models.models import Item as model_Item -from app.core.models.models import ItemAvailability as model_ItemAvailability -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.core.schemas.item import Item, ItemCreate, ItemUpdate -from app.core.schemas.item_availability import ( - ItemAvailability, - ItemAvailabilityCreate, - ItemAvailabilityUpdate, -) -from app.core.schemas.item_base_type import ( +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.core.schemas.item_base_type import ( ItemBaseType, ItemBaseTypeCreate, ItemBaseTypeUpdate, ) -from app.crud.extensions.crud_currency import CRUDCurrency -from app.crud.extensions.crud_item import CRUDItem -from app.crud.extensions.crud_league import CRUDLeague -from app.crud.extensions.crud_modifier import CRUDModifier -from app.crud.extensions.crud_unidentifiedItem import CRUDUnidentifiedItem -from app.crud.user import CRUDUser +from backend_api.app.crud.extensions.crud_currency import CRUDCurrency +from backend_api.app.crud.extensions.crud_item import CRUDItem +from backend_api.app.crud.extensions.crud_league import CRUDLeague +from backend_api.app.crud.extensions.crud_modifier import CRUDModifier +from backend_api.app.crud.extensions.crud_unidentifiedItem import CRUDUnidentifiedItem +from backend_api.app.crud.user import CRUDUser from .base import CRUDBase @@ -25,27 +17,11 @@ CRUD_currency = CRUDCurrency() -CRUD_itemAvailability = CRUDBase[ - model_ItemAvailability, - ItemAvailability, - ItemAvailabilityCreate, - ItemAvailabilityUpdate, -]( - model=model_ItemAvailability, - schema=ItemAvailability, - create_schema=ItemAvailabilityCreate, -) - CRUD_itemBaseType = CRUDBase[ model_ItemBaseType, ItemBaseType, ItemBaseTypeCreate, ItemBaseTypeUpdate ](model=model_ItemBaseType, schema=ItemBaseType, create_schema=ItemBaseTypeCreate) -CRUD_item = CRUDBase[ - model_Item, - Item, - ItemCreate, - ItemUpdate, -](model=model_Item, schema=Item, create_schema=ItemCreate) +CRUD_item = CRUDItem() CRUD_unidentifiedItem = CRUDUnidentifiedItem() diff --git a/src/backend_api/app/crud/base.py b/src/backend_api/app/crud/base.py index efcce7f8..56d4be85 100644 --- a/src/backend_api/app/crud/base.py +++ b/src/backend_api/app/crud/base.py @@ -2,23 +2,22 @@ from itertools import islice from typing import Any, Generic, Literal, TypeVar -from pydantic import BaseModel, TypeAdapter -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session - -# from app.api.params import FilterParams -from app.api.params import FilterParams -from app.exceptions import ( +# from backend_api.app.api.params import FilterParams +from backend_api.app.api.params import FilterParams +from backend_api.app.exceptions import ( ArgValueNotSupportedError, DbObjectDoesNotExistError, DbTooManyItemsDeleteError, ) -from app.exceptions.model_exceptions.db_exception import ( +from backend_api.app.exceptions.model_exceptions.db_exception import ( DbObjectAlreadyExistsError, GeneralDBError, ) -from app.logs.logger import logger -from app.utils.sort_algorithms import sort_with_reference +from backend_api.app.logs.logger import logger +from backend_api.app.utils.sort_algorithms import sort_with_reference +from pydantic import BaseModel, TypeAdapter +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session ModelType = TypeVar("ModelType", bound=Any) SchemaType = TypeVar("SchemaType", bound=Any) diff --git a/src/backend_api/app/crud/extensions/crud_currency.py b/src/backend_api/app/crud/extensions/crud_currency.py index 34de3923..a29f372d 100644 --- a/src/backend_api/app/crud/extensions/crud_currency.py +++ b/src/backend_api/app/crud/extensions/crud_currency.py @@ -1,12 +1,7 @@ -from pydantic import TypeAdapter -from sqlalchemy import select -from sqlalchemy.orm import Session -from sqlalchemy.sql.expression import ColumnElement, and_, func, or_ - -from app.api.params import FilterParams -from app.core.models.models import CurrencyPrice as model_CurrencyPrice -from app.core.models.models import CurrencyType as model_CurrencyType -from app.core.schemas.currency import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.models.models import CurrencyPrice as model_CurrencyPrice +from backend_api.app.core.models.models import CurrencyType as model_CurrencyType +from backend_api.app.core.schemas.currency import ( Currency, CurrencyPrice, CurrencyPriceCreate, @@ -16,7 +11,11 @@ CurrencyTypeCreate, CurrencyTypeUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import select +from sqlalchemy.orm import Session +from sqlalchemy.sql.expression import ColumnElement, and_, func, or_ class CRUDCurrency: diff --git a/src/backend_api/app/crud/extensions/crud_league.py b/src/backend_api/app/crud/extensions/crud_league.py index e1ce7044..acc2d623 100644 --- a/src/backend_api/app/crud/extensions/crud_league.py +++ b/src/backend_api/app/crud/extensions/crud_league.py @@ -1,16 +1,15 @@ from datetime import datetime -from pydantic import TypeAdapter -from sqlalchemy import select -from sqlalchemy.orm import Session - -from app.core.models.models import League as model_League -from app.core.schemas.league import ( +from backend_api.app.core.models.models import League as model_League +from backend_api.app.core.schemas.league import ( League, LeagueCreate, LeagueUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import select +from sqlalchemy.orm import Session class CRUDLeague(CRUDBase[model_League, League, LeagueCreate, LeagueUpdate]): diff --git a/src/backend_api/app/crud/extensions/crud_modifier.py b/src/backend_api/app/crud/extensions/crud_modifier.py index 2bff5eca..7767d3c3 100644 --- a/src/backend_api/app/crud/extensions/crud_modifier.py +++ b/src/backend_api/app/crud/extensions/crud_modifier.py @@ -1,17 +1,16 @@ -from pydantic import TypeAdapter -from sqlalchemy import func, select, update -from sqlalchemy.dialects.postgresql import aggregate_order_by -from sqlalchemy.orm import Session - -from app.core.models.models import Modifier as model_Modifier -from app.core.models.models import ModifierRoll as model_ModifierRoll -from app.core.schemas.modifier import ( +from backend_api.app.core.models.models import Modifier as model_Modifier +from backend_api.app.core.models.models import ModifierRoll as model_ModifierRoll +from backend_api.app.core.schemas.modifier import ( GroupedModifier, Modifier, ModifierCreate, ModifierUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import func, select, update +from sqlalchemy.dialects.postgresql import aggregate_order_by +from sqlalchemy.orm import Session class CRUDModifier( diff --git a/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py b/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py index 0be48d04..ca9b578b 100644 --- a/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py +++ b/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py @@ -1,14 +1,15 @@ -from pydantic import TypeAdapter -from sqlalchemy import delete, select -from sqlalchemy.orm import Session - -from app.core.models.models import UnidentifiedItem as model_UnidentifiedItem -from app.core.schemas.unidentified_item import ( +from backend_api.app.core.models.models import ( + UnidentifiedItem as model_UnidentifiedItem, +) +from backend_api.app.core.schemas.unidentified_item import ( UnidentifiedItem, UnidentifiedItemCreate, UnidentifiedItemUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import delete, select +from sqlalchemy.orm import Session class CRUDUnidentifiedItem( diff --git a/src/backend_api/app/crud/user.py b/src/backend_api/app/crud/user.py index ad60c0a1..43424d11 100644 --- a/src/backend_api/app/crud/user.py +++ b/src/backend_api/app/crud/user.py @@ -1,19 +1,18 @@ from uuid import UUID -from pydantic import EmailStr, TypeAdapter -from sqlalchemy.orm import Session -from sqlalchemy.sql import func, select - -from app.core.models.models import User as model_User -from app.core.schemas import User, UserCreate, UserUpdate -from app.core.schemas.user import UpdatePassword, UsersPublic -from app.core.security import get_password_hash, verify_password -from app.exceptions import ( +from backend_api.app.core.models.models import User as model_User +from backend_api.app.core.schemas import User, UserCreate, UserUpdate +from backend_api.app.core.schemas.user import UpdatePassword, UsersPublic +from backend_api.app.core.security import get_password_hash, verify_password +from backend_api.app.exceptions import ( DbObjectAlreadyExistsError, DbObjectDoesNotExistError, InvalidPasswordError, NewPasswordIsSameError, ) +from pydantic import EmailStr, TypeAdapter +from sqlalchemy.orm import Session +from sqlalchemy.sql import func, select class CRUDUser: diff --git a/src/backend_api/app/exception_handlers.py b/src/backend_api/app/exception_handlers.py index 6af23e43..a7f50510 100644 --- a/src/backend_api/app/exception_handlers.py +++ b/src/backend_api/app/exception_handlers.py @@ -1,5 +1,9 @@ import sys +from backend_api.app.exceptions.model_exceptions.rate_limit_exception import ( + RateLimitExceededError, +) +from backend_api.app.logs.logger import logger from fastapi import Request from fastapi.exception_handlers import http_exception_handler as _http_exception_handler from fastapi.exception_handlers import ( @@ -9,9 +13,6 @@ from fastapi.responses import JSONResponse, PlainTextResponse, Response from slowapi.errors import RateLimitExceeded -from app.exceptions.model_exceptions.rate_limit_exception import RateLimitExceededError -from app.logs.logger import logger - """ Taken from https://medium.com/@roy-pstr/fastapi-server-errors-and-logs-take-back-control-696405437983 diff --git a/src/backend_api/app/exceptions/model_exceptions/cache_exception.py b/src/backend_api/app/exceptions/model_exceptions/cache_exception.py index d3c76342..a4128243 100644 --- a/src/backend_api/app/exceptions/model_exceptions/cache_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/cache_exception.py @@ -1,6 +1,5 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class InvalidCacheUpdateParamsError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/db_exception.py b/src/backend_api/app/exceptions/model_exceptions/db_exception.py index c83119ae..b26e209d 100644 --- a/src/backend_api/app/exceptions/model_exceptions/db_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/db_exception.py @@ -1,10 +1,9 @@ from typing import Any import starlette.status as status - -from app.core.models.models import User as model_User -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import logger +from backend_api.app.core.models.models import User as model_User +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import logger HIDDEN_TABLE_LIST = [model_User.__tablename__] diff --git a/src/backend_api/app/exceptions/model_exceptions/plot_exception.py b/src/backend_api/app/exceptions/model_exceptions/plot_exception.py index a3d02943..0c9a24ed 100644 --- a/src/backend_api/app/exceptions/model_exceptions/plot_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/plot_exception.py @@ -1,7 +1,6 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import logger +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import logger class PlotQueryToDBError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py b/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py index 75ae9a4b..5cc9f478 100644 --- a/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py @@ -1,6 +1,5 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class RateLimitExceededError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/request_exception.py b/src/backend_api/app/exceptions/model_exceptions/request_exception.py index 89fa1cc3..19dbc2e3 100644 --- a/src/backend_api/app/exceptions/model_exceptions/request_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/request_exception.py @@ -1,8 +1,7 @@ from typing import Any import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class InvalidTokenError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/test_exception.py b/src/backend_api/app/exceptions/model_exceptions/test_exception.py index 5dbd8add..b037495d 100644 --- a/src/backend_api/app/exceptions/model_exceptions/test_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/test_exception.py @@ -1,7 +1,6 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import test_logger +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import test_logger class _TestErrorLogBase(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py b/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py index 1a1d9869..0b008af5 100644 --- a/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py @@ -1,6 +1,5 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class UserWithNotEnoughPrivilegesError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/util_exception.py b/src/backend_api/app/exceptions/model_exceptions/util_exception.py index d8c6715f..0695ebfe 100644 --- a/src/backend_api/app/exceptions/model_exceptions/util_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/util_exception.py @@ -1,7 +1,6 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import logger +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import logger class _UtilBaseError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/initial_data.py b/src/backend_api/app/initial_data.py index de228f1c..9e6a2eb8 100644 --- a/src/backend_api/app/initial_data.py +++ b/src/backend_api/app/initial_data.py @@ -1,10 +1,9 @@ import logging +from backend_api.app.core.models.database import engine +from backend_api.app.core.models.init_db import init_db from sqlalchemy.orm import Session -from app.core.models.database import engine -from app.core.models.init_db import init_db - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/backend_api/app/logs/config/__init__.py b/src/backend_api/app/logs/config/__init__.py index 91bf8f59..aa521277 100644 --- a/src/backend_api/app/logs/config/__init__.py +++ b/src/backend_api/app/logs/config/__init__.py @@ -1,2 +1,5 @@ -from app.logs.config.formatters import JSONFormatter, APIMessageFormatter -from app.logs.config.data_filter import SensitiveDataFilter, UnwantedDataFilter +from backend_api.app.logs.config.formatters import JSONFormatter, APIMessageFormatter +from backend_api.app.logs.config.data_filter import ( + SensitiveDataFilter, + UnwantedDataFilter, +) diff --git a/src/backend_api/app/main.py b/src/backend_api/app/main.py index 908dc17c..1d260c12 100644 --- a/src/backend_api/app/main.py +++ b/src/backend_api/app/main.py @@ -1,5 +1,20 @@ from contextlib import asynccontextmanager +from backend_api.app.api.api import api_router +from backend_api.app.core.config import settings +from backend_api.app.core.models.database import async_engine +from backend_api.app.exception_handlers import ( + custom_rate_limit_exceeded_handler, + http_exception_handler, + request_validation_exception_handler, + slow_api_rate_limit_exceeded_handler, + unhandled_exception_handler, +) +from backend_api.app.exceptions.model_exceptions.rate_limit_exception import ( + RateLimitExceededError, +) +from backend_api.app.logs.logger import setup_logging +from backend_api.app.middleware.request_logs import log_request_middleware from fastapi import FastAPI from fastapi.exceptions import RequestValidationError from fastapi.routing import APIRoute @@ -8,20 +23,6 @@ from starlette.middleware.cors import CORSMiddleware from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware -from app.api.api import api_router -from app.core.config import settings -from app.core.models.database import async_engine -from app.exception_handlers import ( - custom_rate_limit_exceeded_handler, - http_exception_handler, - request_validation_exception_handler, - slow_api_rate_limit_exceeded_handler, - unhandled_exception_handler, -) -from app.exceptions.model_exceptions.rate_limit_exception import RateLimitExceededError -from app.logs.logger import setup_logging -from app.middleware.request_logs import log_request_middleware - def custom_generate_unique_id(route: APIRoute) -> str: return f"{route.tags[0]}-{route.name}" diff --git a/src/backend_api/app/middleware/request_logs.py b/src/backend_api/app/middleware/request_logs.py index 53fd22c8..276dd7ef 100644 --- a/src/backend_api/app/middleware/request_logs.py +++ b/src/backend_api/app/middleware/request_logs.py @@ -1,12 +1,11 @@ import http import time -from fastapi import Request - -from app.api.deps import ( +from backend_api.app.api.deps import ( get_user_ip_from_header, ) -from app.logs.logger import logger_request +from backend_api.app.logs.logger import logger_request +from fastapi import Request """ Taken from https://medium.com/@roy-pstr/fastapi-server-errors-and-logs-take-back-control-696405437983 diff --git a/src/backend_api/app/plotting/plotter.py b/src/backend_api/app/plotting/plotter.py index 33955494..8bfe3dc7 100644 --- a/src/backend_api/app/plotting/plotter.py +++ b/src/backend_api/app/plotting/plotter.py @@ -2,47 +2,29 @@ from typing import Any, Generic, TypeVar import pandas as pd -from pydantic import TypeAdapter -from pydantic.fields import FieldInfo -from sqlalchemy import ( - BinaryExpression, - ColumnElement, - Label, - Result, - and_, - case, - desc, - func, - or_, - select, -) -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import InstrumentedAttribute -from sqlalchemy.sql.expression import Select - -from app.core.models.database import engine -from app.core.models.models import ( +from backend_api.app.core.models.database import engine +from backend_api.app.core.models.models import ( CurrencyPrice as model_CurrencyPrice, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( CurrencyType as model_CurrencyType, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Item as model_Item, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( ItemBaseType as model_ItemBaseType, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( ItemModifier as model_ItemModifier, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( UnidentifiedItem as model_UniItem, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( UnidentifiedItem as model_UnidentifiedItem, ) -from app.core.schemas.plot import ( +from backend_api.app.core.schemas.plot import ( BasePlotQuery, IdentifiedPlotQuery, ItemSpecs, @@ -52,12 +34,32 @@ UnidentifiedPlotQuery, WantedModifier, ) -from app.exceptions.model_exceptions.plot_exception import ( +from backend_api.app.exceptions.model_exceptions.plot_exception import ( PlotQueryDataNotFoundError, PlotQueryInvalidError, ) -from app.logs.logger import plot_logger -from app.utils.timing_tracker import async_timing_tracker, sync_timing_tracker +from backend_api.app.logs.logger import plot_logger +from backend_api.app.utils.timing_tracker import ( + async_timing_tracker, + sync_timing_tracker, +) +from pydantic import TypeAdapter +from pydantic.fields import FieldInfo +from sqlalchemy import ( + BinaryExpression, + ColumnElement, + Label, + Result, + and_, + case, + desc, + func, + or_, + select, +) +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.sql.expression import Select Q = TypeVar("Q", bound=PlotQuery) diff --git a/src/backend_api/app/tests/test_real_env/api/real_env_base.py b/src/backend_api/app/tests/test_real_env/api/real_env_base.py index 8450d5c9..76c3773e 100644 --- a/src/backend_api/app/tests/test_real_env/api/real_env_base.py +++ b/src/backend_api/app/tests/test_real_env/api/real_env_base.py @@ -1,13 +1,12 @@ from collections.abc import AsyncGenerator +from backend_api.app.core.config import settings +from backend_api.app.logs.logger import setup_logging +from backend_api.app.logs.logger import test_logger as logger +from backend_api.app.main import app +from backend_api.app.tests.utils.utils import get_superuser_token_headers from httpx import AsyncClient -from app.core.config import settings -from app.logs.logger import setup_logging -from app.logs.logger import test_logger as logger -from app.main import app -from app.tests.utils.utils import get_superuser_token_headers - class RealEnvBase: """Inheritable test real env base class with help functions""" diff --git a/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py b/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py index 7e8bcf8a..9103df5a 100644 --- a/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py +++ b/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py @@ -3,8 +3,8 @@ from json import JSONDecodeError from typing import Any -from app.logs.logger import test_logger as logger -from app.tests.test_real_env.api.real_env_base import RealEnvBase +from backend_api.app.logs.logger import test_logger as logger +from backend_api.app.tests.test_real_env.api.real_env_base import RealEnvBase class TestPlotRealEnv(RealEnvBase): diff --git a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py index 860da99d..1e3f18fa 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py +++ b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py @@ -3,22 +3,25 @@ from typing import Any import pytest -from httpx import AsyncClient, Response -from sqlalchemy.orm import Session - -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_delete_return_msg, ) -from app.core.cache.user_cache import UserCache -from app.core.config import settings -from app.crud.base import CRUDBase, ModelType -from app.exceptions import ( +from backend_api.app.core.cache.user_cache import UserCache +from backend_api.app.core.config import settings +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.exceptions import ( DbObjectDoesNotExistError, ) -from app.exceptions.model_exceptions.db_exception import DbObjectAlreadyExistsError -from app.exceptions.model_exceptions.request_exception import InvalidTokenError -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import is_courotine_function +from backend_api.app.exceptions.model_exceptions.db_exception import ( + DbObjectAlreadyExistsError, +) +from backend_api.app.exceptions.model_exceptions.request_exception import ( + InvalidTokenError, +) +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import is_courotine_function +from httpx import AsyncClient, Response +from sqlalchemy.orm import Session @pytest.mark.usefixtures("clear_db", autouse=True) diff --git a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py index 74e1a5a7..7820a406 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py +++ b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py @@ -2,13 +2,14 @@ from typing import Any import pytest +from backend_api.app.core.config import settings +from backend_api.app.tests.test_simulating_env.api.api_test_rate_limit_base import ( + TestRateLimitBase, +) +from backend_api.app.tests.utils.rate_limit import RateLimitPerTimeInterval from slowapi import Limiter from sqlalchemy.orm import Session -from app.core.config import settings -from app.tests.test_simulating_env.api.api_test_rate_limit_base import TestRateLimitBase -from app.tests.utils.rate_limit import RateLimitPerTimeInterval - @pytest.mark.usefixtures("clear_db", autouse=True) @pytest.mark.skipif( diff --git a/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py b/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py index 8cab5d94..4080dabe 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py +++ b/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py @@ -3,8 +3,8 @@ from typing import Any from unittest.mock import patch -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.rate_limit import RateLimitPerTimeInterval +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.rate_limit import RateLimitPerTimeInterval class TestRateLimitBase(BaseTest): diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py index 5ed1a9df..54e80637 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py @@ -3,28 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import currency_prefix, league_prefix -from app.api.routes.currency import get_currency -from app.core.config import settings -from app.core.models.models import Currency, League -from app.crud import CRUD_currency -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.currency import ( +from backend_api.app.api.routes import currency_prefix, league_prefix +from backend_api.app.api.routes.currency import get_currency +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Currency, League +from backend_api.app.crud import CRUD_currency +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.currency import ( create_random_currency_dict, generate_random_currency, ) -from app.tests.utils.rate_limit import ( +from backend_api.app.tests.utils.rate_limit import ( RateLimitPerTimeInterval, get_function_decorator_rate_limit_per_time_interval, ) -from app.tests.utils.utils import ( +from backend_api.app.tests.utils.utils import ( get_model_table_name, get_model_unique_identifier, ) +from fastapi import Response +from httpx import AsyncClient + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py index e0170825..bd7fffde 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py @@ -3,25 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient -from sqlalchemy.orm import Session - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import ( +from backend_api.app.api.routes import ( currency_prefix, item_base_type_prefix, item_prefix, ) -from app.core.config import settings -from app.core.models.models import Currency, Item, ItemBaseType -from app.crud import CRUD_item -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item import ( +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Currency, Item, ItemBaseType +from backend_api.app.crud import CRUD_item +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item import ( create_random_item_dict, generate_random_item, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from fastapi import Response +from httpx import AsyncClient +from sqlalchemy.orm import Session + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py index 7adb4a17..b02c677c 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py @@ -3,25 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import item_base_type_prefix -from app.api.routes.item_base_type import get_item_base_type -from app.core.config import settings -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.crud import CRUD_itemBaseType -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item_base_type import ( +from backend_api.app.api.routes import item_base_type_prefix +from backend_api.app.api.routes.item_base_type import get_item_base_type +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.crud import CRUD_itemBaseType +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item_base_type import ( create_random_item_base_type_dict, generate_random_item_base_type, ) -from app.tests.utils.rate_limit import ( +from backend_api.app.tests.utils.rate_limit import ( RateLimitPerTimeInterval, get_function_decorator_rate_limit_per_time_interval, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from fastapi import Response +from httpx import AsyncClient + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py index e695dfe2..26eeb702 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py @@ -1,29 +1,31 @@ from collections.abc import Awaitable, Callable import pytest -from sqlalchemy.orm import Session - -from app.api.routes import ( +from backend_api.app.api.routes import ( currency_prefix, item_base_type_prefix, item_modifier_prefix, item_prefix, modifier_prefix, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Currency, Item, ItemBaseType, ItemModifier, Modifier, ) -from app.crud import CRUD_itemModifier -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item_modifier import ( +from backend_api.app.crud import CRUD_itemModifier +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item_modifier import ( create_random_item_modifier_dict, generate_random_item_modifier, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from sqlalchemy.orm import Session @pytest.fixture(scope="module") @@ -114,14 +116,16 @@ async def create_object(db: Session) -> dict: @pytest.fixture(scope="module") -def object_generator_func_w_deps() -> Callable[ - [], - tuple[ - dict, - ItemModifier, - list[dict | Item | ItemBaseType | Currency | Modifier], - ], -]: +def object_generator_func_w_deps() -> ( + Callable[ + [], + tuple[ + dict, + ItemModifier, + list[dict | Item | ItemBaseType | Currency | Modifier], + ], + ] +): def generate_random_item_modifier_w_deps( db, ) -> Callable[ diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py index 249220af..d8336251 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py @@ -3,21 +3,24 @@ import pytest import pytest_asyncio +from backend_api.app.api.routes import item_prefix +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import League +from backend_api.app.crud import CRUD_item +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item import ( + create_random_item_dict, + generate_random_item, +) +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) from fastapi import Response from httpx import AsyncClient from sqlalchemy.orm import Session import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import item_prefix -from app.core.config import settings -from app.core.models.models import League -from app.crud import CRUD_item -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item import ( - create_random_item_dict, - generate_random_item, -) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py index 34afc785..db039b82 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py @@ -1,16 +1,17 @@ from collections.abc import Awaitable import pytest -from fastapi import Response -from httpx import AsyncClient - -from app.api.routes.login import login_access_session, login_prefix -from app.core.config import settings -from app.tests.test_simulating_env.api.api_test_rate_limit_base import TestRateLimitBase -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.rate_limit import ( +from backend_api.app.api.routes.login import login_access_session, login_prefix +from backend_api.app.core.config import settings +from backend_api.app.tests.test_simulating_env.api.api_test_rate_limit_base import ( + TestRateLimitBase, +) +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.rate_limit import ( get_function_decorator_rate_limit_per_time_interval, ) +from fastapi import Response +from httpx import AsyncClient @pytest.mark.usefixtures("clear_db", autouse=True) diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py index 1c51af94..61888dc8 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py @@ -3,25 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import modifier_prefix -from app.api.routes.modifier import get_modifier -from app.core.config import settings -from app.core.models.models import Modifier -from app.crud import CRUD_modifier -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.modifier import ( +from backend_api.app.api.routes import modifier_prefix +from backend_api.app.api.routes.modifier import get_modifier +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Modifier +from backend_api.app.crud import CRUD_modifier +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.modifier import ( create_random_modifier_dict, generate_random_modifier, ) -from app.tests.utils.rate_limit import ( +from backend_api.app.tests.utils.rate_limit import ( RateLimitPerTimeInterval, get_function_decorator_rate_limit_per_time_interval, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from fastapi import Response +from httpx import AsyncClient + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py index b06f2243..516cf9bb 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py @@ -5,17 +5,20 @@ from typing import Any import pytest +from backend_api.app.api.routes.plot import plot_prefix +from backend_api.app.core.config import settings +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.tests.test_simulating_env.api.api_test_rate_limit_base import ( + TestRateLimitBase, +) +from backend_api.app.tests.utils.model_utils.plot import ( + create_minimal_random_plot_query_dict, +) +from backend_api.app.tests.utils.rate_limit import RateLimitPerTimeInterval from fastapi import Response from httpx import AsyncClient from sqlalchemy.orm import Session -from app.api.routes.plot import plot_prefix -from app.core.config import settings -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.tests.test_simulating_env.api.api_test_rate_limit_base import TestRateLimitBase -from app.tests.utils.model_utils.plot import create_minimal_random_plot_query_dict -from app.tests.utils.rate_limit import RateLimitPerTimeInterval - @pytest.mark.usefixtures("clear_db", autouse=True) class TestPlotAPI(TestRateLimitBase): diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py index 286a704f..e54213ce 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py @@ -2,14 +2,15 @@ from unittest.mock import patch import pytest +from backend_api.app.api.api_message_util import ( + get_failed_send_challenge_request_error_msg, +) +from backend_api.app.api.routes import turnstile_prefix +from backend_api.app.core.config import settings +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import create_random_ip from httpx import AsyncClient -from app.api.api_message_util import get_failed_send_challenge_request_error_msg -from app.api.routes import turnstile_prefix -from app.core.config import settings -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import create_random_ip - @pytest.mark.usefixtures("clear_db", autouse=True) class TestTurnstileAPI(BaseTest): diff --git a/src/backend_api/app/tests/test_simulating_env/base_test.py b/src/backend_api/app/tests/test_simulating_env/base_test.py index 8bd46ca2..8d3f4fa8 100644 --- a/src/backend_api/app/tests/test_simulating_env/base_test.py +++ b/src/backend_api/app/tests/test_simulating_env/base_test.py @@ -3,15 +3,14 @@ from collections.abc import Awaitable, Callable from typing import Any -from sqlalchemy.inspection import inspect -from sqlalchemy.orm import Session - -from app.core.models.database import insp -from app.crud.base import ( +from backend_api.app.core.models.database import insp +from backend_api.app.crud.base import ( CRUDBase, ModelType, ) -from app.tests.utils.utils import get_extract_functions +from backend_api.app.tests.utils.utils import get_extract_functions +from sqlalchemy.inspection import inspect +from sqlalchemy.orm import Session class BaseTest: diff --git a/src/backend_api/app/tests/test_simulating_env/conftest.py b/src/backend_api/app/tests/test_simulating_env/conftest.py index 01386908..0b9695b5 100644 --- a/src/backend_api/app/tests/test_simulating_env/conftest.py +++ b/src/backend_api/app/tests/test_simulating_env/conftest.py @@ -6,6 +6,23 @@ import pytest import pytest_asyncio +from backend_api.app.api.deps import get_async_db, get_db +from backend_api.app.core.cache.cache import cache +from backend_api.app.core.cache.user_cache import UserCache, UserCacheTokenType +from backend_api.app.core.config import settings +from backend_api.app.core.models.init_db import init_db +from backend_api.app.core.rate_limit.rate_limiters import limiter_ip, limiter_user +from backend_api.app.main import app as actual_app +from backend_api.app.tests.test_simulating_env.setup_test_database import ( + ASYNC_TEST_DATABASE_URL, + TEST_DATABASE_URL, +) +from backend_api.app.tests.utils.database_utils import ( + clear_all_tables, + mock_src_database_for_test_db, +) +from backend_api.app.tests.utils.user import authentication_token_from_email +from backend_api.app.tests.utils.utils import get_superuser_token_headers from fastapi import FastAPI from fastapi.testclient import TestClient from httpx import ASGITransport, AsyncClient @@ -19,24 +36,6 @@ ) from sqlalchemy.orm import Session -from app.api.deps import get_async_db, get_db -from app.core.cache.cache import cache -from app.core.cache.user_cache import UserCache, UserCacheTokenType -from app.core.config import settings -from app.core.models.init_db import init_db -from app.core.rate_limit.rate_limiters import limiter_ip, limiter_user -from app.main import app as actual_app -from app.tests.test_simulating_env.setup_test_database import ( - ASYNC_TEST_DATABASE_URL, - TEST_DATABASE_URL, -) -from app.tests.utils.database_utils import ( - clear_all_tables, - mock_src_database_for_test_db, -) -from app.tests.utils.user import authentication_token_from_email -from app.tests.utils.utils import get_superuser_token_headers - @pytest.fixture(autouse=True) def app() -> Generator[FastAPI, None, None]: diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py index 56b3bb7b..7899f092 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py @@ -1,13 +1,13 @@ from collections.abc import Callable, Generator import pytest +from backend_api.app.core.models.database import engine +from backend_api.app.crud import CRUD_currency +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.currency import generate_random_currency from sqlalchemy.orm import Session import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.database import engine -from app.crud import CRUD_currency -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.currency import generate_random_currency @pytest.fixture(scope="session") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py index 14dc94f6..18f196fd 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py @@ -1,16 +1,16 @@ from collections.abc import Callable import pytest - -import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.models import Currency, Item, ItemBaseType -from app.crud import ( +from backend_api.app.core.models.models import Currency, Item, ItemBaseType +from backend_api.app.crud import ( CRUD_currency, CRUD_item, CRUD_itemBaseType, ) -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.item import generate_random_item +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.item import generate_random_item + +import app.tests.test_simulating_env.crud.crud_test_base as test_crud @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py index f7c47c46..4c941b39 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py @@ -1,12 +1,14 @@ from collections.abc import Callable import pytest +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.crud import CRUD_itemBaseType +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.item_base_type import ( + generate_random_item_base_type, +) import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.crud import CRUD_itemBaseType -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.item_base_type import generate_random_item_base_type @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py index 7e22cf35..ad20344b 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py @@ -1,23 +1,24 @@ from collections.abc import Callable import pytest - -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Currency, Item, ItemBaseType, ItemModifier, Modifier, ) -from app.crud import ( +from backend_api.app.crud import ( CRUD_currency, CRUD_item, CRUD_itemBaseType, CRUD_itemModifier, CRUD_modifier, ) -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.item_modifier import generate_random_item_modifier +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.item_modifier import ( + generate_random_item_modifier, +) @pytest.fixture(scope="module") @@ -31,14 +32,16 @@ def is_hypertable() -> bool: @pytest.fixture(scope="module") -def object_generator_func_w_deps() -> Callable[ - [], - tuple[ - dict, - ItemModifier, - list[dict | Item | ItemBaseType | Currency | Modifier], - ], -]: +def object_generator_func_w_deps() -> ( + Callable[ + [], + tuple[ + dict, + ItemModifier, + list[dict | Item | ItemBaseType | Currency | Modifier], + ], + ] +): def generate_random_item_modifier_w_deps( db, ) -> Callable[ diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py index e5dccb9c..28031cf6 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py @@ -1,13 +1,13 @@ from collections.abc import Callable, Generator import pytest +from backend_api.app.core.models.database import engine +from backend_api.app.crud import CRUD_league +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.league import generate_random_league from sqlalchemy.orm import Session import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.database import engine -from app.crud import CRUD_league -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.league import generate_random_league @pytest.fixture(scope="session") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py index 3bbc724c..9933bf66 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py @@ -1,9 +1,9 @@ import pytest +from backend_api.app.crud import CRUD_modifier +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.modifier import generate_random_modifier import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.crud import CRUD_modifier -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.modifier import generate_random_modifier @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py index 430268d2..a0bce5d5 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py @@ -1,4 +1,4 @@ -from app.tests.test_simulating_env.crud.crud_test_user import TestUserCRUD +from backend_api.app.tests.test_simulating_env.crud.crud_test_user import TestUserCRUD class TestUserCRUD(TestUserCRUD): diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py index dc4d1231..1c8cb1c9 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py @@ -1,18 +1,17 @@ from collections.abc import Callable import pytest -from sqlalchemy.orm import Session - -from app.crud.base import ( +from backend_api.app.crud.base import ( CRUDBase, ModelType, ) -from app.exceptions.model_exceptions.db_exception import ( +from backend_api.app.exceptions.model_exceptions.db_exception import ( DbObjectAlreadyExistsError, DbObjectDoesNotExistError, ) -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import get_ignore_keys +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import get_ignore_keys +from sqlalchemy.orm import Session @pytest.mark.usefixtures("clear_db", autouse=True) diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py index 5421109b..150d5099 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py @@ -1,12 +1,11 @@ import pytest +from backend_api.app.core.schemas import UserCreate, UserUpdate +from backend_api.app.core.security import verify_password +from backend_api.app.crud import CRUD_user as crud +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import random_email, random_lower_string from sqlalchemy.orm import Session -from app.core.schemas import UserCreate, UserUpdate -from app.core.security import verify_password -from app.crud import CRUD_user as crud -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import random_email, random_lower_string - @pytest.mark.usefixtures("clear_db", autouse=True) class TestUserCRUD(BaseTest): diff --git a/src/backend_api/app/tests/test_simulating_env/setup_test_database.py b/src/backend_api/app/tests/test_simulating_env/setup_test_database.py index 979dffd6..822852ec 100644 --- a/src/backend_api/app/tests/test_simulating_env/setup_test_database.py +++ b/src/backend_api/app/tests/test_simulating_env/setup_test_database.py @@ -1,7 +1,6 @@ +from backend_api.app.core.config import settings from pydantic import PostgresDsn -from app.core.config import settings - TEST_DATABASE_URL: PostgresDsn | None = str(settings.TEST_DATABASE_URI) ASYNC_TEST_DATABASE_URL: PostgresDsn | None = str(settings.ASYNC_TEST_DATABASE_URI) diff --git a/src/backend_api/app/tests/utils/database_utils.py b/src/backend_api/app/tests/utils/database_utils.py index e41185f1..58b4e61d 100644 --- a/src/backend_api/app/tests/utils/database_utils.py +++ b/src/backend_api/app/tests/utils/database_utils.py @@ -1,10 +1,9 @@ +from backend_api.app.core.config import settings +from backend_api.app.core.models.database import engine as src_db_engine +from backend_api.app.core.models.models import User from sqlalchemy import Engine, MetaData, delete from sqlalchemy.exc import SQLAlchemyError -from app.core.config import settings -from app.core.models.database import engine as src_db_engine -from app.core.models.models import User - src_db_metadata = MetaData() test_db_metadata = MetaData() diff --git a/src/backend_api/app/tests/utils/model_utils/__init__.py b/src/backend_api/app/tests/utils/model_utils/__init__.py index 98c8f19e..17de52a6 100644 --- a/src/backend_api/app/tests/utils/model_utils/__init__.py +++ b/src/backend_api/app/tests/utils/model_utils/__init__.py @@ -1,20 +1,20 @@ -from app.tests.utils.model_utils.currency import ( +from backend_api.app.tests.utils.model_utils.currency import ( create_random_currency_dict, generate_random_currency, ) -from app.tests.utils.model_utils.item_base_type import ( +from backend_api.app.tests.utils.model_utils.item_base_type import ( create_random_item_base_type_dict, generate_random_item_base_type, ) -from app.tests.utils.model_utils.item_modifier import ( +from backend_api.app.tests.utils.model_utils.item_modifier import ( create_random_item_modifier_dict, generate_random_item_modifier, ) -from app.tests.utils.model_utils.item import ( +from backend_api.app.tests.utils.model_utils.item import ( create_random_item_dict, generate_random_item, ) -from app.tests.utils.model_utils.modifier import ( +from backend_api.app.tests.utils.model_utils.modifier import ( create_random_modifier_dict, generate_random_modifier, ) diff --git a/src/backend_api/app/tests/utils/model_utils/currency.py b/src/backend_api/app/tests/utils/model_utils/currency.py index 7b733aba..9095e3c3 100644 --- a/src/backend_api/app/tests/utils/model_utils/currency.py +++ b/src/backend_api/app/tests/utils/model_utils/currency.py @@ -1,14 +1,14 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import Currency, League -from app.core.schemas import CurrencyCreate -from app.tests.utils.model_utils.league import generate_random_league -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import Currency, League +from backend_api.app.core.schemas import CurrencyCreate +from backend_api.app.tests.utils.model_utils.league import generate_random_league +from backend_api.app.tests.utils.utils import ( random_float, random_int, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud async def create_random_currency_dict( diff --git a/src/backend_api/app/tests/utils/model_utils/item.py b/src/backend_api/app/tests/utils/model_utils/item.py index c9dde2dd..7ec0ee66 100644 --- a/src/backend_api/app/tests/utils/model_utils/item.py +++ b/src/backend_api/app/tests/utils/model_utils/item.py @@ -1,18 +1,20 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import Currency, Item, ItemBaseType, League -from app.core.schemas.item import ItemCreate -from app.tests.utils.model_utils.currency import generate_random_currency -from app.tests.utils.model_utils.item_base_type import generate_random_item_base_type -from app.tests.utils.model_utils.league import generate_random_league -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import Currency, Item, ItemBaseType, League +from backend_api.app.core.schemas.item import ItemCreate +from backend_api.app.tests.utils.model_utils.currency import generate_random_currency +from backend_api.app.tests.utils.model_utils.item_base_type import ( + generate_random_item_base_type, +) +from backend_api.app.tests.utils.model_utils.league import generate_random_league +from backend_api.app.tests.utils.utils import ( random_bool, random_float, random_int, random_json, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud async def create_random_item_dict( diff --git a/src/backend_api/app/tests/utils/model_utils/item_base_type.py b/src/backend_api/app/tests/utils/model_utils/item_base_type.py index 05b8c4c6..a4fe06a2 100644 --- a/src/backend_api/app/tests/utils/model_utils/item_base_type.py +++ b/src/backend_api/app/tests/utils/model_utils/item_base_type.py @@ -1,9 +1,9 @@ +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.core.schemas.item_base_type import ItemBaseTypeCreate +from backend_api.app.tests.utils.utils import random_int, random_lower_string from sqlalchemy.orm import Session from app import crud -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.core.schemas.item_base_type import ItemBaseTypeCreate -from app.tests.utils.utils import random_int, random_lower_string def create_random_item_base_type_dict() -> dict: diff --git a/src/backend_api/app/tests/utils/model_utils/item_modifier.py b/src/backend_api/app/tests/utils/model_utils/item_modifier.py index d28735bf..8bc85c78 100644 --- a/src/backend_api/app/tests/utils/model_utils/item_modifier.py +++ b/src/backend_api/app/tests/utils/model_utils/item_modifier.py @@ -1,7 +1,4 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Currency, Item, ItemBaseType, @@ -9,10 +6,13 @@ League, Modifier, ) -from app.core.schemas.item_modifier import ItemModifierCreate -from app.tests.utils.model_utils.item import generate_random_item -from app.tests.utils.model_utils.modifier import generate_random_modifier -from app.tests.utils.utils import random_float, random_int +from backend_api.app.core.schemas.item_modifier import ItemModifierCreate +from backend_api.app.tests.utils.model_utils.item import generate_random_item +from backend_api.app.tests.utils.model_utils.modifier import generate_random_modifier +from backend_api.app.tests.utils.utils import random_float, random_int +from sqlalchemy.orm import Session + +from app import crud async def create_random_item_modifier_dict( diff --git a/src/backend_api/app/tests/utils/model_utils/league.py b/src/backend_api/app/tests/utils/model_utils/league.py index aa2fc308..8e7ebdee 100644 --- a/src/backend_api/app/tests/utils/model_utils/league.py +++ b/src/backend_api/app/tests/utils/model_utils/league.py @@ -1,13 +1,13 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import League -from app.core.schemas import LeagueCreate -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import League +from backend_api.app.core.schemas import LeagueCreate +from backend_api.app.tests.utils.utils import ( random_datetime, random_float, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud def create_random_league_dict() -> dict: diff --git a/src/backend_api/app/tests/utils/model_utils/modifier.py b/src/backend_api/app/tests/utils/model_utils/modifier.py index e78f0941..0573e021 100644 --- a/src/backend_api/app/tests/utils/model_utils/modifier.py +++ b/src/backend_api/app/tests/utils/model_utils/modifier.py @@ -1,14 +1,14 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import Modifier -from app.core.schemas.modifier import ModifierCreate -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import Modifier +from backend_api.app.core.schemas.modifier import ModifierCreate +from backend_api.app.tests.utils.utils import ( random_bool, random_float, random_int, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud def create_random_modifier_dict() -> dict: diff --git a/src/backend_api/app/tests/utils/model_utils/plot.py b/src/backend_api/app/tests/utils/model_utils/plot.py index f71e91e6..0a644851 100644 --- a/src/backend_api/app/tests/utils/model_utils/plot.py +++ b/src/backend_api/app/tests/utils/model_utils/plot.py @@ -1,16 +1,15 @@ from typing import Any -from sqlalchemy.orm import Session - -from app.core.models.models import ( +from backend_api.app.core.models.models import ( League as model_League, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Modifier as model_Modifier, ) -from app.tests.utils.model_utils.item_modifier import ( +from backend_api.app.tests.utils.model_utils.item_modifier import ( generate_random_item_modifier, ) +from sqlalchemy.orm import Session async def create_minimal_random_plot_query_dict(db: Session) -> dict[str, Any]: diff --git a/src/backend_api/app/tests/utils/user.py b/src/backend_api/app/tests/utils/user.py index 0de73fb9..dd2bbbec 100644 --- a/src/backend_api/app/tests/utils/user.py +++ b/src/backend_api/app/tests/utils/user.py @@ -1,12 +1,11 @@ +from backend_api.app.core.config import settings +from backend_api.app.core.schemas import User, UserCreate, UserUpdate +from backend_api.app.crud import CRUD_user +from backend_api.app.tests.utils.utils import random_email, random_lower_string from httpx import AsyncClient from pydantic import EmailStr from sqlalchemy.orm import Session -from app.core.config import settings -from app.core.schemas import User, UserCreate, UserUpdate -from app.crud import CRUD_user -from app.tests.utils.utils import random_email, random_lower_string - async def user_authentication_headers( *, async_client: AsyncClient, email: str, password: str diff --git a/src/backend_api/app/tests/utils/utils.py b/src/backend_api/app/tests/utils/utils.py index 405a6ed6..f65d5d27 100644 --- a/src/backend_api/app/tests/utils/utils.py +++ b/src/backend_api/app/tests/utils/utils.py @@ -5,12 +5,11 @@ from inspect import iscoroutinefunction from typing import Any +from backend_api.app.core.config import settings +from backend_api.app.crud.base import ModelType from httpx import AsyncClient from sqlalchemy import inspect -from app.core.config import settings -from app.crud.base import ModelType - def random_lower_string(*, small_string: bool | None = None) -> str: """Generate a random lowercase string. diff --git a/src/backend_api/app/utils/timing_tracker.py b/src/backend_api/app/utils/timing_tracker.py index fac64659..a1319163 100644 --- a/src/backend_api/app/utils/timing_tracker.py +++ b/src/backend_api/app/utils/timing_tracker.py @@ -3,7 +3,7 @@ from time import perf_counter from typing import ParamSpec, TypeVar -from app.logs.logger import logger +from backend_api.app.logs.logger import logger Param = ParamSpec("Param") RetType = TypeVar("RetType") diff --git a/src/backend_api/app/utils/user.py b/src/backend_api/app/utils/user.py index ce66e523..a74f11ed 100644 --- a/src/backend_api/app/utils/user.py +++ b/src/backend_api/app/utils/user.py @@ -3,12 +3,11 @@ from typing import Any import emails # type: ignore +from backend_api.app.core.config import settings +from backend_api.app.logs.logger import logger from jinja2 import Template from pydantic import EmailStr -from app.core.config import settings -from app.logs.logger import logger - @dataclass class EmailData: diff --git a/src/backend_api/app/validation/turnstile_validation.py b/src/backend_api/app/validation/turnstile_validation.py index f79ab0e1..df86f263 100644 --- a/src/backend_api/app/validation/turnstile_validation.py +++ b/src/backend_api/app/validation/turnstile_validation.py @@ -1,11 +1,12 @@ import httpx # Use httpx instead of requests +from backend_api.app.api.api_message_util import ( + get_failed_send_challenge_request_error_msg, +) +from backend_api.app.core.config import settings +from backend_api.app.core.schemas import TurnstileQuery, TurnstileResponse from fastapi import HTTPException from pydantic import TypeAdapter -from app.api.api_message_util import get_failed_send_challenge_request_error_msg -from app.core.config import settings -from app.core.schemas import TurnstileQuery, TurnstileResponse - class ValidateTurnstileRequest: def __init__(self): diff --git a/src/backend_api/scripts/prestart.sh b/src/backend_api/scripts/prestart.sh index 03cd269b..00a49cf8 100644 --- a/src/backend_api/scripts/prestart.sh +++ b/src/backend_api/scripts/prestart.sh @@ -4,12 +4,12 @@ set -e set -x # Let the DB start -python /app/app/backend_pre_start.py +python /app/backend_api/app/backend_pre_start.py # Run migrations alembic upgrade head # Create initial data in DB -python /app/app/initial_data.py +python /app/backend_api/app/initial_data.py -fastapi run --reload "app/main.py" +fastapi run --reload "backend_api/app/main.py" From bf77d044c09ee052e1ce62de63b5ed49b3d41468 Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 26 Sep 2026 18:12:07 -0230 Subject: [PATCH 09/16] #830 Fixed bug with backend api logger util paths --- src/backend_api/app/logs/config/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend_api/app/logs/config/config.yml b/src/backend_api/app/logs/config/config.yml index a81e1f6b..cbc5abbb 100644 --- a/src/backend_api/app/logs/config/config.yml +++ b/src/backend_api/app/logs/config/config.yml @@ -5,7 +5,7 @@ formatters: format: "[%(name)s|%(levelname)s|%(module)s]: %(asctime)s: %(message)s" datefmt: "%Y-%m-%dT%H:%M:%S%z" json: - (): app.logs.config.JSONFormatter + (): backend_api.app.logs.config.JSONFormatter fmt_keys: logger: name level: levelname @@ -13,9 +13,9 @@ formatters: message: message filters: sensitive_data_filter: - (): app.logs.config.SensitiveDataFilter + (): backend_api.app.logs.config.SensitiveDataFilter unwanted_data_filter: - (): app.logs.config.UnwantedDataFilter + (): backend_api.app.logs.config.UnwantedDataFilter handlers: stdout: class: logging.StreamHandler @@ -34,7 +34,7 @@ handlers: level: DEBUG filters: [sensitive_data_filter] formatter: simple - filename: "app/logs/backend-api.log" + filename: "backend_api/app/logs/backend-api.log" maxBytes: 504857600 # 500 MB backupCount: 2 loggers: From 64299a2bff5c83b12dd6eff7fadd8e35632f437e Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Wed, 30 Sep 2026 16:20:47 -0230 Subject: [PATCH 10/16] #830 Fixed bug with backend not syncing during docker watch --- src/docker-compose.override.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index 5adbb8b3..b940236f 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -64,7 +64,7 @@ services: watch: - path: ./backend_api action: sync - target: /app + target: /app/backend_api ignore: - .venv - uv.lock From 674eb1009897854195b2536afaabf104e767d1f6 Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Wed, 30 Sep 2026 16:25:53 -0230 Subject: [PATCH 11/16] #830 Separated backend api and data retrieval caches --- src/.env | 6 ++---- src/backend_api/app/core/cache/cache.py | 2 +- src/backend_api/app/core/config.py | 6 +++--- src/backend_api/app/core/rate_limit/rate_limiters.py | 4 ++-- .../data_retrieval_app/external_data_retrieval/cache.py | 4 +++- .../data_retrieval_app/external_data_retrieval/config.py | 6 +++--- src/docker-compose.yml | 3 +-- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/.env b/src/.env index a78e29dc..42f9365c 100644 --- a/src/.env +++ b/src/.env @@ -38,8 +38,8 @@ TURNSTILE_SECRET_KEY=1x0000000000000000000000000000000AA # Data retrieval MANUAL_NEXT_CHANGE_ID=True -# Change ID Retrieved 28.06.2026: -NEXT_CHANGE_ID=3176140940-3103957281-3029294684-3370770619-3261964453 +# Change ID Retrieved 13.09.2026: +NEXT_CHANGE_ID=3273127146-3199406693-3124831624-3474177832-3363107859 POE_PUBLIC_STASHES_AUTH_TOKEN=changethis # Enables the public stashes creation script in data_ret/app/tests/scripts. # Set value "True" to activate: @@ -68,9 +68,7 @@ OATH_ACC_TOKEN_CONTACT_EMAIL=changethis # Redis Cache REDIS_PORT=6379 REDIS_SERVER=cache -REDIS_CACHE=0 REDIS_PASSWORD=changethis -REDIS_URI=redis://:${REDIS_PASSWORD}@${REDIS_SERVER}:${REDIS_PORT}/${REDIS_CACHE} # Vector VECTOR_PORT=8686 diff --git a/src/backend_api/app/core/cache/cache.py b/src/backend_api/app/core/cache/cache.py index 9faca654..9945306d 100644 --- a/src/backend_api/app/core/cache/cache.py +++ b/src/backend_api/app/core/cache/cache.py @@ -1,4 +1,4 @@ import redis.asyncio as Redis from backend_api.app.core.config import settings -cache = Redis.from_url(str(settings.CACHE_URI), decode_responses=True) +cache = Redis.from_url(str(settings.API_CACHE_URI), decode_responses=True) diff --git a/src/backend_api/app/core/config.py b/src/backend_api/app/core/config.py index 971b2cf0..be359fa2 100644 --- a/src/backend_api/app/core/config.py +++ b/src/backend_api/app/core/config.py @@ -83,18 +83,18 @@ def ASYNC_DATABASE_URI(self) -> PostgresDsn: REDIS_PORT: int = 6379 REDIS_SERVER: str - REDIS_CACHE: str = str(0) + REDIS_BACKEND_API: int = 0 REDIS_PASSWORD: str = "" @computed_field # type: ignore[prop-decorator] @property - def CACHE_URI(self) -> AnyUrl: + def API_CACHE_URI(self) -> AnyUrl: return MultiHostUrl.build( scheme="redis", password=self.REDIS_PASSWORD, host=self.REDIS_SERVER, port=self.REDIS_PORT, - path=self.REDIS_CACHE, + path=f"{self.REDIS_BACKEND_API}", ) SMTP_TLS: bool = True diff --git a/src/backend_api/app/core/rate_limit/rate_limiters.py b/src/backend_api/app/core/rate_limit/rate_limiters.py index a65381c0..6f3f34ce 100644 --- a/src/backend_api/app/core/rate_limit/rate_limiters.py +++ b/src/backend_api/app/core/rate_limit/rate_limiters.py @@ -23,7 +23,7 @@ def default_limit_provider() -> list[str]: limiter_user = Limiter( key_func=get_username_by_request, default_limits=default_limit_provider(), - storage_uri=str(settings.CACHE_URI), + storage_uri=str(settings.API_CACHE_URI), headers_enabled=True, enabled=settings.RATE_LIMIT, ) @@ -32,7 +32,7 @@ def default_limit_provider() -> list[str]: limiter_ip = Limiter( key_func=get_user_ip_from_header, default_limits=default_limit_provider(), - storage_uri=str(settings.CACHE_URI), + storage_uri=str(settings.API_CACHE_URI), headers_enabled=True, enabled=settings.RATE_LIMIT, ) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py index 1ebbb923..97a98caa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py @@ -7,7 +7,9 @@ @contextmanager def get_cache(): - cache = redis.from_url(str(settings.CACHE_URI), decode_responses=True) + cache = redis.from_url( + str(settings.DATA_RETRIEVAL_CACHE_URI), decode_responses=True + ) try: yield cache diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py index 72c8dd42..721fc0c9 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py @@ -26,18 +26,18 @@ def BACKEND_BASE_URL(self) -> HttpUrl: REDIS_PORT: int = 6379 REDIS_SERVER: str - REDIS_CACHE: str = str(0) + REDIS_BACKEND_DATA_RETRIEVAL: int = 1 REDIS_PASSWORD: str = "" @computed_field # type: ignore[prop-decorator] @property - def CACHE_URI(self) -> AnyUrl: + def DATA_RETRIEVAL_CACHE_URI(self) -> AnyUrl: return MultiHostUrl.build( scheme="redis", password=self.REDIS_PASSWORD, host=self.REDIS_SERVER, port=self.REDIS_PORT, - path=self.REDIS_CACHE, + path=f"{self.REDIS_BACKEND_DATA_RETRIEVAL}", ) MANUAL_NEXT_CHANGE_ID: bool diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 9e01a867..8c8851f5 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -41,7 +41,6 @@ services: - node.labels.${STACK_NAME?Variable not set}.pom-cache-data == true environment: REDIS_PASSWORD: ${REDIS_PASSWORD?Variable not set} - REDIS_CACHE: ${REDIS_CACHE?Variable not set} volumes: - pom-cache-data:/data healthcheck: @@ -181,7 +180,7 @@ services: volumes: pom-db-data: - name: "pom-db-default-data" + name: "830-test-pom-db-default-data" pom-cache-data: name: "pom-cache-data" pgadmin-data: From 6ad20c630dc01106a473b2d3f0a1b5c8862a2fee Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Wed, 30 Sep 2026 16:30:28 -0230 Subject: [PATCH 12/16] #830 Fixed some modifier table bugs --- .../23d7asdb13df_split_modifier_table.py | 2 +- src/backend_api/app/core/models/models.py | 4 ++++ src/backend_api/app/core/schemas/modifier.py | 5 +++-- .../app/crud/extensions/crud_modifier.py | 20 +++++++++++++------ .../modifier/modifier_data_depositor.py | 3 +++ .../modifier/modifier_processing_modules.py | 4 ++-- 6 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py index 33133594..36810f3d 100644 --- a/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py +++ b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py @@ -101,7 +101,7 @@ def upgrade() -> None: batch_op.drop_column("maxRoll") batch_op.drop_column("textRolls") batch_op.create_primary_key("pk_modifier", ["modifierId"]) - batch_op.create_unique_constraint("uq_effect") + batch_op.create_unique_constraint("uq_effect", ["effect"]) with op.batch_alter_table("modifier_roll") as batch_op: batch_op.create_foreign_key( diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index d89c610a..1f632d30 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -321,6 +321,10 @@ class ItemModifier(Base): SmallInteger, nullable=False, ) + instance: Mapped[int] = mapped_column( + SmallInteger, + nullable=False, + ) # a modifier can appear multiple times on an item (eg. forbidden shako) roll: Mapped[float | None] = mapped_column( Float(4), ) diff --git a/src/backend_api/app/core/schemas/modifier.py b/src/backend_api/app/core/schemas/modifier.py index a8ff0ee2..9bd86412 100644 --- a/src/backend_api/app/core/schemas/modifier.py +++ b/src/backend_api/app/core/schemas/modifier.py @@ -61,8 +61,9 @@ class Modifier(ModifierBase): class GroupedModifier(Modifier): - rolls: list[ModifierRoll] = _pydantic.Field(default_factory=list) + rolls: list[ModifierRoll] -class ModifierUpdate(GroupedModifier): +class ModifierUpdate(Modifier): + rolls: list[ModifierRoll] | None = None effect: str | None = None diff --git a/src/backend_api/app/crud/extensions/crud_modifier.py b/src/backend_api/app/crud/extensions/crud_modifier.py index 7767d3c3..02d3d281 100644 --- a/src/backend_api/app/crud/extensions/crud_modifier.py +++ b/src/backend_api/app/crud/extensions/crud_modifier.py @@ -56,16 +56,24 @@ async def update(self, db: Session, *, modifier: ModifierUpdate): update_modifier_stmt = ( update(model_Modifier) .where(model_Modifier.modifierId == modifier.modifierId) - .values(**modifier.model_dump(exclude_unset=True, exclude_none=True)) + .values( + **modifier.model_dump( + exclude_unset=True, exclude_none=True, exclude={"rolls"} + ) + ) ) db.execute(update_modifier_stmt) - modifier_rolls = [ - {"modifierId": modifier.modifierId, **roll.model_dump(exclude_unset=True)} - for roll in modifier.rolls - ] - if modifier_rolls: + if modifier.rolls is not None: + modifier_rolls = [ + { + "modifierId": modifier.modifierId, + **roll.model_dump(exclude_unset=True), + } + for roll in modifier.rolls + ] db.execute(update(model_ModifierRoll), modifier_rolls) + db.commit() async def get_grouped_modifiers(self, db: Session) -> list[GroupedModifier]: diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py index 1713f1b2..94fc60ef 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py @@ -87,6 +87,9 @@ def _check_for_updates( update_regex = True # regex contains text rolls if roll_need_update: + if updated_modifier.rolls is None: + updated_modifier.rolls = list[ModifierRoll]() + updated_modifier.rolls.append(updated_roll) need_update = True diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py index 12464a18..631dd1a2 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py @@ -8,9 +8,9 @@ def make_regex(self, effect: str, rolls: list[ModifierRoll]) -> str: regex = effect.replace("+", "[+-]") for roll in rolls: if roll.textRolls is not None: - effect = effect.replace("#", f"({roll.textRolls})", 1) + regex = regex.replace("#", f"({"|".join(roll.textRolls)})", 1) else: - effect = effect.replace("#", r"([0-9]*[.]?[0-9]+)", 1) + regex = regex.replace("#", r"([0-9]*[.]?[0-9]+)", 1) regex = re.sub(r"increased|reduced", "(increased|reduced)", regex) From 5773c6ef42895c3c0fd59d4bfcd5d4903bcb063d Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Wed, 30 Sep 2026 16:32:41 -0230 Subject: [PATCH 13/16] #830 Fixed some backend api/db item/item_modifier issues --- .../dd4abb2b6541_optimize_item_table.py | 23 +- src/backend_api/app/api/routes/item.py | 71 ++--- .../app/api/routes/item_availability.py | 38 --- src/backend_api/app/core/models/models.py | 9 +- src/backend_api/app/core/schemas/__init__.py | 2 + src/backend_api/app/core/schemas/item.py | 82 +++-- .../app/core/schemas/item_modifier.py | 4 +- .../app/crud/extensions/crud_item.py | 300 ++++++++++++++++++ 8 files changed, 425 insertions(+), 104 deletions(-) delete mode 100644 src/backend_api/app/api/routes/item_availability.py create mode 100644 src/backend_api/app/crud/extensions/crud_item.py diff --git a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py index 01fac0eb..a78fbab3 100644 --- a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py +++ b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py @@ -240,6 +240,13 @@ def upgrade() -> None: # TODO optimize when writing the plotting query sa.Index("ix_item_availability_itemId_validFrom", "itemId", "validFrom"), sa.UniqueConstraint("itemId", "validFrom"), + sa.CheckConstraint( + """ + item_availability."validTo" IS NULL + OR item_availability."validTo" > item_availability."validFrom" + """, + name="check_positive_duration", + ), ) op.execute(""" INSERT INTO item_availability ( @@ -269,6 +276,9 @@ def upgrade() -> None: i."itemId"::INT, im."modifierId", im.position, + ROW_NUMBER() OVER ( + PARTITION BY i."itemId", im."modifierId", im.position + )::SMALLINT AS instance, im.roll FROM item i LEFT JOIN item_modifier im @@ -284,11 +294,12 @@ def upgrade() -> None: batch_op.alter_column("itemId", nullable=False) batch_op.alter_column("modifierId", nullable=False) batch_op.alter_column("position", nullable=False) + batch_op.alter_column("instance", nullable=False) batch_op.alter_column( "roll", existing_type=sa.Float(), type_=sa.Float(4), nullable=True ) batch_op.create_primary_key( - "item_modifier_pkey", ["itemId", "modifierId", "position"] + "item_modifier_pkey", ["itemId", "modifierId", "position", "instance"] ) batch_op.create_foreign_key( "fk_item_modifier_item", @@ -306,11 +317,11 @@ def upgrade() -> None: ondelete="RESTRICT", onupdate="CASCADE", ) - batch_op.create_index( - "ix_item_modifier_modifierId_itemId", - ["modifierId", "itemId"], - unique=False, - ) + # batch_op.create_index( + # "ix_item_modifier_modifierId_itemId", + # ["modifierId", "itemId"], + # unique=False, + # ) def downgrade() -> None: diff --git a/src/backend_api/app/api/routes/item.py b/src/backend_api/app/api/routes/item.py index 5c2f121d..87d47725 100644 --- a/src/backend_api/app/api/routes/item.py +++ b/src/backend_api/app/api/routes/item.py @@ -3,15 +3,11 @@ import backend_api.app.core.schemas as schemas from backend_api.app.api.deps import ( get_current_active_superuser, - get_current_active_user, get_db, ) from backend_api.app.api.params import FilterParams -from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings -from backend_api.app.core.rate_limit.rate_limiters import apply_user_rate_limits from backend_api.app.crud import CRUD_item -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy import text +from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session router = APIRouter() @@ -20,36 +16,6 @@ item_prefix = "item" -@router.get( - "/latest_item_id/", - response_model=int | None, - tags=["latest_item_id"], - dependencies=[Depends(get_current_active_user)], -) -@apply_user_rate_limits( - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_MINUTE, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_HOUR, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_DAY, -) -async def get_latest_item_id( - request: Request, # noqa: ARG001 - response: Response, # noqa: ARG001 - db: Session = Depends(get_db), -): - """ - Get the latest "itemId" - - Can only be used safely on an empty table or directly after an insertion. - """ - - result = db.execute(text("""SELECT MAX("itemId") FROM item""")).fetchone() - if not result or not result[0]: - return None - - return int(result[0]) - - @router.get( "/", response_model=schemas.Item | list[schemas.Item], @@ -72,12 +38,11 @@ async def get_all_items( @router.post( "/", - response_model=schemas.ItemCreate | list[schemas.ItemCreate] | None, + response_model=list[schemas.Item], dependencies=[Depends(get_current_active_superuser)], ) async def create_item( - item: schemas.ItemCreate | list[schemas.ItemCreate], - return_nothing: bool | None = None, + items: list[schemas.ItemCreate], db: Session = Depends(get_db), ): """ @@ -86,4 +51,32 @@ async def create_item( Returns the created item or list of items. """ - return await CRUD_item.create(db=db, obj_in=item, return_nothing=return_nothing) + return await CRUD_item.create_items(db=db, new_items=items) + + +@router.patch( + "/availability/", + response_model=list[schemas.ItemAvailability], + dependencies=[Depends(get_current_active_superuser)], +) +async def patch_expired_availability( + expired_availability: list[schemas.ItemAvailabilityExpired], + db: Session = Depends(get_db), +): + return await CRUD_item.patch_expired_availability( + db, expired_availability=expired_availability + ) + + +@router.put( + "/availability/", + response_model=list[schemas.ItemAvailability], + dependencies=[Depends(get_current_active_superuser)], +) +async def update_availability( + updated_availability: list[schemas.ItemAvailabilityUpdated], + db: Session = Depends(get_db), +): + return await CRUD_item.update_availability( + db, updated_availability=updated_availability + ) diff --git a/src/backend_api/app/api/routes/item_availability.py b/src/backend_api/app/api/routes/item_availability.py deleted file mode 100644 index 723f34e1..00000000 --- a/src/backend_api/app/api/routes/item_availability.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Annotated - -import backend_api.app.core.schemas as schemas -from backend_api.app.api.deps import ( - get_current_active_superuser, - get_db, -) -from backend_api.app.api.params import FilterParams -from backend_api.app.crud import CRUD_itemAvailability -from fastapi import APIRouter, Depends, Query -from sqlalchemy.orm import Session - -router = APIRouter() - - -item_availability_prefix = "itemAvailability" - - -@router.get( - "/", - response_model=schemas.ItemAvailability | list[schemas.ItemAvailability], - dependencies=[Depends(get_current_active_superuser)], -) -async def get_all_item_modifiers( - filter_params: Annotated[FilterParams, Query()], - db: Session = Depends(get_db), -): - """ - Get all available items. - - Returns a list of all available items. - """ - - all_item_availability = await CRUD_itemAvailability.get( - db=db, filter_params=filter_params - ) - - return all_item_availability diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index 1f632d30..f2001f61 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -89,7 +89,7 @@ class ItemAvailability(Base): Integer, primary_key=True, autoincrement=True ) itemId: Mapped[str] = mapped_column( - Text, + Integer, ForeignKey("item.itemId", ondelete="CASCADE", onupdate="CASCADE"), nullable=False, ) @@ -109,6 +109,13 @@ class ItemAvailability(Base): __table_args__ = ( Index("ix_item_id_valid_from", "itemId", "validFrom"), UniqueConstraint("itemId", "validFrom"), + CheckConstraint( + """ + item_availability."validTo" IS NULL + OR item_availability."validTo" > item_availability."validFrom" + """, + name="check_positive_duration", + ), ) diff --git a/src/backend_api/app/core/schemas/__init__.py b/src/backend_api/app/core/schemas/__init__.py index d8536777..7af82163 100644 --- a/src/backend_api/app/core/schemas/__init__.py +++ b/src/backend_api/app/core/schemas/__init__.py @@ -14,6 +14,8 @@ from .item import ( Item, ItemAvailability, + ItemAvailabilityExpired, + ItemAvailabilityUpdated, ItemCreate, ItemQuery, ItemUpdate, diff --git a/src/backend_api/app/core/schemas/item.py b/src/backend_api/app/core/schemas/item.py index 40421c98..dd278ab1 100644 --- a/src/backend_api/app/core/schemas/item.py +++ b/src/backend_api/app/core/schemas/item.py @@ -1,4 +1,5 @@ import pydantic as _pydantic +from backend_api.app.core.schemas.item_modifier import ItemModifierCreate class Influences(_pydantic.BaseModel): @@ -11,16 +12,18 @@ class Influences(_pydantic.BaseModel): # Shared item props -class _BaseItem(_pydantic.BaseModel): - model_config = _pydantic.ConfigDict(from_attributes=True) +class ItemBase(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict( + from_attributes=True, extra="ignore", populate_by_name=True + ) - gameItemId: str - leagueId: int + gameItemId: str = _pydantic.Field(alias="game_item_id") + leagueId: int = _pydantic.Field(alias="league_id") - firstObserved: int + firstObserved: int = _pydantic.Field(alias="first_observed") name: str - itemBaseTypeId: int + itemBaseTypeId: int = _pydantic.Field(alias="item_base_type_id") ilvl: int rarity: str @@ -35,26 +38,69 @@ class _BaseItem(_pydantic.BaseModel): tangled: bool | None = None -# Properties to receive on item creation -class ItemCreate(_BaseItem): - pass +class ItemPrice(_pydantic.BaseModel): + currencyId: int + currencyAmount: float + isAsync: bool | None = None + + +class ItemCreate(_pydantic.BaseModel): + item: ItemBase + price: ItemPrice + modifiers: list[ItemModifierCreate] # Properties to receive on update -class ItemUpdate(_BaseItem): +class ItemUpdate(ItemBase): pass -# Properties shared by models stored in DB -class ItemInDBBase(_BaseItem): +# Properties to return to client +class Item(ItemBase): itemId: int -# Properties to return to client -class Item(ItemInDBBase): - pass +class ItemQuery(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict( + extra="ignore", + populate_by_name=True, + ) + gameItemId: str + league: str -# Properties stored in DB -class ItemInDB(ItemInDBBase): - pass + +# Shared item props +class _BaseItemAvailability(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + itemId: int + + currencyId: int + currencyAmount: float + + validFrom: int + validTo: int | None = None + + isAsync: bool | None = None + + +class ItemAvailabilityExpired(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + gameItemId: str + leagueId: int + + validTo: int + + +class ItemAvailabilityUpdated(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + gameItemId: str + leagueId: int + + price: ItemPrice + + validFrom: int + + +class ItemAvailability(_BaseItemAvailability): + availabilityId: int diff --git a/src/backend_api/app/core/schemas/item_modifier.py b/src/backend_api/app/core/schemas/item_modifier.py index 39642e45..f00b55d6 100644 --- a/src/backend_api/app/core/schemas/item_modifier.py +++ b/src/backend_api/app/core/schemas/item_modifier.py @@ -11,7 +11,7 @@ class ItemModifier(_pydantic.BaseModel): roll: float | None = None -class ModifierRoll(_pydantic.BaseModel): +class ItemModifierRoll(_pydantic.BaseModel): position: int roll: float | None = None @@ -19,4 +19,4 @@ class ModifierRoll(_pydantic.BaseModel): class ItemModifierCreate(_pydantic.BaseModel): modifierId: int - rolls: list[ModifierRoll] | None = None + rolls: list[ItemModifierRoll] diff --git a/src/backend_api/app/crud/extensions/crud_item.py b/src/backend_api/app/crud/extensions/crud_item.py new file mode 100644 index 00000000..a97c810a --- /dev/null +++ b/src/backend_api/app/crud/extensions/crud_item.py @@ -0,0 +1,300 @@ +from collections import defaultdict +from collections.abc import Iterable +from typing import Any + +from backend_api.app.core.models.models import Item as model_Item +from backend_api.app.core.models.models import ( + ItemAvailability as model_ItemAvailability, +) +from backend_api.app.core.models.models import ItemModifier as model_ItemModifier +from backend_api.app.core.schemas.item import ( + Item, + ItemAvailability, + ItemAvailabilityExpired, + ItemAvailabilityUpdated, + ItemBase, + ItemCreate, +) +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import ( + Boolean, + Float, + Integer, + String, + column, + delete, + select, + update, + values, +) +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session + + +class CRUDItem: + def __init__(self): + self.crud_base = CRUDBase[model_Item, Item, Any, Any]( + model=model_Item, schema=Item, create_schema=None + ) + + self.item_base_adapter = TypeAdapter(list[ItemBase]) + + self.item_validate = TypeAdapter(list[Item]).validate_python + + self.item_availability_validate = TypeAdapter( + list[ItemAvailability] + ).validate_python + + async def get(self, *args, **kwargs): + return await self.crud_base.get(*args, **kwargs) + + async def _incoming_to_database_map( + self, db: Session, *, incoming: Iterable[tuple[str, int]] + ) -> dict[tuple[str, int], int]: + incoming_items_table = values( + column("game_item_id", String), + column("league_id", Integer), + name="incoming_items", + ).data(incoming) + + incoming_to_database_ids_stmt = select( + model_Item.itemId, + model_Item.gameItemId, + model_Item.leagueId, + ).join( + incoming_items_table, + (model_Item.leagueId == incoming_items_table.c.league_id) + & (model_Item.gameItemId == incoming_items_table.c.game_item_id), + ) + + incoming_items = db.execute(incoming_to_database_ids_stmt).mappings().all() + + return { + (incoming.gameItemId, incoming.leagueId): incoming.itemId + for incoming in incoming_items + } + + async def create_items( + self, db: Session, *, new_items: list[ItemCreate] + ) -> list[Item]: + if not new_items: + return [] + + # Deduplicates update_availability, only caring about the latest price + latest_by_item = { + (item.item.gameItemId, item.item.leagueId): item for item in new_items + } + new_items = list(latest_by_item.values()) + + incoming_to_database = await self._incoming_to_database_map( + db, incoming=latest_by_item.keys() + ) + + truely_new_items = list[ItemCreate]() + existing_items = list[ItemCreate]() + + for item in new_items: + game_item_id = item.item.gameItemId + league_id = item.item.leagueId + + db_item_id = incoming_to_database.get((game_item_id, league_id)) + if db_item_id is None: + truely_new_items.append(item) + else: + existing_items.append(item) + + db_items = list[model_Item]() + for new_item in truely_new_items: + db_items.append(model_Item(**new_item.item.model_dump())) + + db.add_all(db_items) + db.flush() + + db_item_availability = list[model_ItemAvailability]() + + instance_count = defaultdict[tuple[int, int], int](int) + db_item_modifiers = list[model_ItemModifier]() + for db_item, new_item in zip(db_items, truely_new_items, strict=True): + for modifier in new_item.modifiers: + key = (db_item.itemId, modifier.modifierId) + for roll in modifier.rolls: + db_item_modifiers.append( + model_ItemModifier( + itemId=db_item.itemId, + modifierId=modifier.modifierId, + position=roll.position, + instance=instance_count[key], + roll=roll.roll, + ) + ) + # a modifier can appear multiple times on an item (eg. forbidden shako) + instance_count[key] += 1 + + db_item_availability.append( + model_ItemAvailability( + itemId=db_item.itemId, + currencyId=new_item.price.currencyId, + currencyAmount=new_item.price.currencyAmount, + validFrom=new_item.item.firstObserved, + isAsync=new_item.price.isAsync, + ) + ) + + db.add_all(db_item_modifiers) + + for existing_item in existing_items: + game_item_id = existing_item.item.gameItemId + league_id = existing_item.item.leagueId + + db_item_id = incoming_to_database[(game_item_id, league_id)] + + db_item_availability.append( + model_ItemAvailability( + itemId=db_item_id, + currencyId=existing_item.price.currencyId, + currencyAmount=existing_item.price.currencyAmount, + validFrom=existing_item.item.firstObserved, + isAsync=existing_item.price.isAsync, + ) + ) + + db.add_all(db_item_availability) + db.flush() + db.commit() + + return self.item_validate(db_items) + + async def update_availability( + self, db: Session, *, updated_availability: list[ItemAvailabilityUpdated] + ) -> list[ItemAvailability]: + if not updated_availability: + return [] + # Deduplicates update_availability, only caring about the latest price + latest_by_item = { + (item.gameItemId, item.leagueId): item for item in updated_availability + } + updated_availability = list(latest_by_item.values()) + + incoming_to_database = await self._incoming_to_database_map( + db, incoming=latest_by_item.keys() + ) + + incoming_table = values( + column("item_id", Integer), + column("currency_id", Integer), + column("currency_amount", Float), + column("valid_from", Integer), + column("is_async", Boolean), + name="incoming", + ).data( + [ + ( + incoming_to_database[(item.gameItemId, item.leagueId)], + item.price.currencyId, + item.price.currencyAmount, + item.validFrom, + item.price.isAsync, + ) + for item in updated_availability + ] + ) + # 1. Delete existing same-hour availability + db.execute( + delete(model_ItemAvailability).where( + model_ItemAvailability.itemId == incoming_table.c.item_id, + model_ItemAvailability.validFrom == incoming_table.c.valid_from, + ) + ) + + # 2. Close current availability + db.execute( + update(model_ItemAvailability) + .where( + model_ItemAvailability.itemId == incoming_table.c.item_id, + model_ItemAvailability.validTo.is_(None), + ) + .values( + validTo=incoming_table.c.valid_from, + ) + ) + + # 3. Insert the new availability + new_availability = ( + db.execute( + insert(model_ItemAvailability).from_select( + [ + "itemId", + "currencyId", + "currencyAmount", + "validFrom", + "isAsync", + ], + select( + incoming_table.c.item_id, + incoming_table.c.currency_id, + incoming_table.c.currency_amount, + incoming_table.c.valid_from, + incoming_table.c.is_async, + ), + ) + ) + .scalars() + .all() + ) + + db.commit() + return self.item_availability_validate(new_availability) + + async def patch_expired_availability( + self, db: Session, *, expired_availability: list[ItemAvailabilityExpired] + ) -> list[ItemAvailability]: + if not expired_availability: + return [] + incoming_to_database = await self._incoming_to_database_map( + db, + incoming=[ + (item.gameItemId, item.leagueId) for item in expired_availability + ], + ) + + expired_table = values( + column("item_id", Integer), + column("valid_to", Integer), + name="expired", + ).data( + [ + ( + incoming_to_database[(item.gameItemId, item.leagueId)], + item.validTo, + ) + for item in expired_availability + ] + ) + # 1. Delete existing same-hour availability + db.execute( + delete(model_ItemAvailability).where( + model_ItemAvailability.itemId == expired_table.c.item_id, + model_ItemAvailability.validFrom == expired_table.c.valid_to, + ) + ) + + # 2. Close current availability + updated_availability = ( + db.execute( + update(model_ItemAvailability) + .where( + model_ItemAvailability.itemId == expired_table.c.item_id, + model_ItemAvailability.validTo.is_(None), + ) + .values( + validTo=expired_table.c.valid_from, + ) + ) + .scalars() + .all() + ) + + db.commit() + + return self.item_availability_validate(updated_availability) From 46c7fe274d525a9460283e3b54fc76968c18803d Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Wed, 30 Sep 2026 16:48:30 -0230 Subject: [PATCH 14/16] #830 Working identified unique item filtering, transformation, and inserting :D --- .../currency/currency_type_data_depositor.py | 2 +- .../external_data_retrieval/config.py | 2 + .../data_retrieval/currency_api_handler.py | 72 +- .../data_retrieval/poe_api_handler.py | 73 +- .../data_retrieval/schemas/base.py | 8 + .../data_retrieval/schemas/external/poe.py | 84 +- .../data_retrieval/schemas/internal/cache.py | 18 + .../schemas/internal/categorized.py | 47 + .../external_data_retrieval/detectors/base.py | 41 +- .../detectors/detector_controller.py | 271 ++++-- .../detectors/unique_detector.py | 162 ++-- .../external_data_retrieval/main.py | 128 +-- .../transforming_data/roll_processor.py | 283 ++---- .../transform_currency_api_data.py | 46 +- .../transform_poe_api_data.py | 833 ++++++------------ .../data_retrieval_app/logs/logger.py | 5 +- 16 files changed, 888 insertions(+), 1187 deletions(-) create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py create mode 100644 src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py index 814aa1f4..98dc13b4 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py @@ -153,7 +153,7 @@ def _process_data( if not found_chaos: processed_currencies.append( - CurrencyTypeCreate(name="Chaos Orb", tradeName="Chaos") + CurrencyTypeCreate(name="Chaos Orb", tradeName="chaos") ) return processed_currencies diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py index 721fc0c9..707a0cd0 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py @@ -53,6 +53,8 @@ def DATA_RETRIEVAL_CACHE_URI(self) -> AnyUrl: MINI_BATCH_SIZE: int = 30 N_CHECKPOINTS_PER_TRANSFORMATION: int = 10 + MAX_MIRROR_PRICE: int = 5 + MAX_TIME_PER_MINI_BATCH: int = 3 * 60 diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py index f2133290..3cf20e8c 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py @@ -1,4 +1,3 @@ -from collections import defaultdict from collections.abc import Sequence from backend_api.app.core.schemas.currency import Currency, CurrencyType @@ -16,7 +15,7 @@ from data_retrieval_app.pom_api_authentication import ( get_superuser_token_headers, ) -from data_retrieval_app.utils import get_data_safe +from data_retrieval_app.utils import send_request_safe class CurrencyAPIHandler: @@ -34,7 +33,8 @@ def __init__(self, url: str) -> None: self.transformer = TransformCurrencyAPIData(self.currency_types) def _get_currency_types(self) -> dict[str, CurrencyType]: - response = get_data_safe( + response = send_request_safe( + "get", self.currency_url + "type/", headers=self.pom_auth_headers, logger=logger, @@ -46,58 +46,54 @@ def _get_currency_types(self) -> dict[str, CurrencyType]: return {currency_type.name: currency_type for currency_type in currency_types} def _get_latest_currencies( - self, league_ids: Sequence[int] - ) -> dict[str, list[Currency]]: - response = get_data_safe( + self, leagues: Sequence[League] + ) -> dict[tuple[int, str], Currency]: + response = send_request_safe( + "get", self.currency_url + "price/latest/", - params={"league_ids": league_ids}, + params={"league_ids": [league.leagueId for league in leagues]}, headers=self.pom_auth_headers, logger=logger, ) - currencies = TypeAdapter(list[Currency]).validate_python(response.json()) + currency_list = TypeAdapter(list[Currency]).validate_python(response.json()) - trade_name_to_currencies = defaultdict[str, list[Currency]](list) - for currency in currencies: - trade_name_to_currencies[currency.tradeName].append(currency) + currencies = dict[tuple[int, str], Currency]() + for currency in currency_list: + currencies[(currency.leagueId, currency.tradeName)] = currency - return dict(trade_name_to_currencies) + return currencies def _remove_old_currencies( self, - trade_name_to_currencies: dict[str, list[Currency]], + currencies: dict[tuple[int, str], Currency], leagues: list[League], current_hours: dict[int, int], ) -> list[League]: """ Modifies latest currencies inplace """ - leagues_needs_new_data = list[League]() - if not trade_name_to_currencies: + if not currencies: # No prior data is available return leagues - for currencies in trade_name_to_currencies.values(): - for league in leagues: - for currency in currencies[:]: - if league.leagueId != currency.leagueId: - continue + leagues_needs_new_data = set[int]() + for (league_id, _), currency in currencies.items(): + current_hour = current_hours[league_id] + currency_hour = currency.createdHoursSinceLaunch + if currency_hour != current_hour: + leagues_needs_new_data.add(league_id) - current_hour = current_hours[currency.leagueId] - if currency.createdHoursSinceLaunch == current_hour: - continue - - currencies.remove(currency) - - if league not in leagues_needs_new_data: - leagues_needs_new_data.append(league) - - return leagues_needs_new_data + return [ + league for league in leagues if league.leagueId in leagues_needs_new_data + ] def _get_new_data(self, leagues: list[League]) -> list[ExchangeRatioItem]: currency_items = list[ExchangeRatioItem]() for league in leagues: - response = get_data_safe( - self.url.format(league=league.name.replace(" ", "+")), logger=logger + response = send_request_safe( + "get", + self.url.format(league=league.name.replace(" ", "+")), + logger=logger, ) response_json = response.json() @@ -112,25 +108,23 @@ def _get_new_data(self, leagues: list[League]) -> list[ExchangeRatioItem]: def get_currency_data( self, leagues: list[League], current_hours: dict[int, int] - ) -> dict[str, list[Currency]]: + ) -> dict[tuple[int, str], Currency]: """ Returns a dict mapping trade name to list of the same currency per league """ - id_to_league = {league.leagueId: league for league in leagues} - trade_name_to_currencies = self._get_latest_currencies(id_to_league.keys()) - print(trade_name_to_currencies) + currencies = self._get_latest_currencies(leagues) leagues_needs_new_data = self._remove_old_currencies( - trade_name_to_currencies, leagues, current_hours + currencies, leagues, current_hours ) if leagues_needs_new_data: exchange_ratios = self._get_new_data(leagues_needs_new_data) - trade_name_to_currencies.update( + currencies.update( self.transformer.transform_and_insert(exchange_ratios, current_hours) ) - return dict(trade_name_to_currencies) + return currencies def store_data_to_csv(self, path: str) -> None: """ diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py index 742007a6..66618fab 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py @@ -5,15 +5,21 @@ from collections.abc import Iterator from concurrent.futures import Future, ThreadPoolExecutor from queue import Empty, Full, Queue -from typing import Any, Literal +from typing import Literal import httpx import pandas as pd import redis from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter -from data_retrieval_app.external_data_retrieval.cache import get_cache from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + Stash, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.categorized import ( + OrganizedItemsByCategory, +) from data_retrieval_app.external_data_retrieval.data_retrieval.utils import ( ByteResponse, RateLimiterThreadSafe, @@ -22,11 +28,8 @@ DetectorController, ) from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( - UniqueArmourDetector, - UniqueJewelDetector, - UniqueJewelleryDetector, - UniqueUnidentifiedDetector, - UniqueWeaponDetector, + # UnidentifiedUniqueDetector, + UniqueDetector, ) from data_retrieval_app.external_data_retrieval.utils import ( ProgramTooSlowException, @@ -66,14 +69,10 @@ def __init__( if detector_controller is None: detector_controller = DetectorController( [ - UniqueArmourDetector(), - UniqueJewelDetector(), - UniqueJewelleryDetector(), - UniqueWeaponDetector(), - UniqueUnidentifiedDetector(), + UniqueDetector(), + # UnidentifiedUniqueDetector(), ], leagues, - get_cache(), ) self.url = url logger.debug("Url set to: " + self.url) @@ -84,6 +83,8 @@ def __init__( self.detector_controller = detector_controller + self.stashes_adapter = TypeAdapter(list[Stash]) + self.skip_program_too_slow = False logger.info("PoEAPIHandler successfully initialized.") @@ -271,9 +272,9 @@ def initialize_data_stream_threads( return futures @sync_timing_tracker - def _read_stream(self) -> tuple[list[Any], str | None]: + def _read_stream(self) -> tuple[list[Stash], str | None]: i = 0 - stashes = [] + stashes = list[Stash]() next_change_id = None while i < self.mini_batch_size: try: @@ -290,7 +291,7 @@ def _read_stream(self) -> tuple[list[Any], str | None]: continue obj = json.loads(pending.response.decode("utf-8")) - stashes.extend(obj["stashes"]) + stashes.extend(self.stashes_adapter.validate_python(obj["stashes"])) self.response_queue.task_done() next_change_id = pending.next_change_id @@ -299,20 +300,14 @@ def _read_stream(self) -> tuple[list[Any], str | None]: return stashes, next_change_id - @sync_timing_tracker - def _process_stream(self, stashes: list) -> pd.DataFrame: - logger.info("Stashes are ready for processing") - self.detector_controller._filter_never_used(stashes) - - logger.info("Finished processing the data, waiting for more") - # return wanted_df - - def _gather_n_checkpoints(self, n: int) -> tuple[pd.DataFrame | None, str | None]: - df = None + def _gather_n_checkpoints(self, n: int) -> tuple[OrganizedItemsByCategory, str]: + all_organized_items = OrganizedItemsByCategory() for _ in range(n): start_time = time.perf_counter() stashes, next_change_id = self._read_stream() - wanted_df = self._process_stream(stashes) + organized_items = self.detector_controller.filter_stashes( + stashes, self.redis_cache + ) end_time = time.perf_counter() time_per_mini_batch = end_time - start_time @@ -325,27 +320,23 @@ def _gather_n_checkpoints(self, n: int) -> tuple[pd.DataFrame | None, str | None # Does not allow a batch to take longer than 2 minutes raise ProgramTooSlowException - if wanted_df.empty: - continue - - if df is None: - df = wanted_df - else: - df = pd.concat((df, wanted_df)) + all_organized_items.extend(organized_items) - return df, next_change_id + return all_organized_items, next_change_id - def dump_stream(self) -> Iterator[tuple[pd.DataFrame, str | None]]: - time.sleep(5) # Waits for the listening threads to have time to start up. + def dump_stream( + self, cache: redis.Redis + ) -> Iterator[tuple[OrganizedItemsByCategory, str]]: + self.redis_cache = cache + # time.sleep(5) # Waits for the listening threads to have time to start up. while True: logger.info("Waiting for data from the stream") - df, next_change_id = self._gather_n_checkpoints( + organized_items, next_change_id = self._gather_n_checkpoints( n=settings.N_CHECKPOINTS_PER_TRANSFORMATION, ) - if df is None: + if organized_items.is_empty(): logger.info("Found no data") continue logger.info("Finished processing the stream, entering transformation phase") - yield df.reset_index(), next_change_id - del df + yield organized_items, next_change_id logger.info("Finished transformation phase") diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py new file mode 100644 index 00000000..7843e04f --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel, ConfigDict + + +class PoeModel(BaseModel): + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + ) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py index 5067633a..5a1dcafa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py @@ -1,37 +1,25 @@ from __future__ import annotations -from typing import Any +import re +from typing import TYPE_CHECKING -from pydantic import BaseModel, ConfigDict, Field +from pydantic import Field +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.base import ( + PoeModel, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItem, + CacheStash, +) -class PoeModel(BaseModel): - model_config = ConfigDict( - extra="ignore", - populate_by_name=True, +if TYPE_CHECKING: + # To avoid circular imports, while maintaining type hinting + from data_retrieval_app.external_data_retrieval.detectors.detector_controller import ( + DetectorController, ) -class CacheItem(PoeModel): - id: str - note: str - - -class CacheStash(PoeModel): - league: str - items: list[CacheItem] - - -class ItemProperty(PoeModel): - name: str - values: list[list[Any]] - display_mode: int | None = Field(default=None, alias="displayMode") - progress: float | None = None - type: int | None = None - suffix: str | None = None - icon: str | None = None - - class ItemModFlags(PoeModel): fractured: bool | None = None mutated: bool | None = None @@ -50,7 +38,7 @@ class Extended(PoeModel): suffixes: int | None = None -class Item(PoeModel): +class PoeItem(PoeModel): # Basic identity / display id: str | None = None name: str @@ -58,7 +46,7 @@ class Item(PoeModel): base_type: str = Field(alias="baseType") rarity: str | None = None - # Item state + # PoeItem state verified: bool identified: bool corrupted: bool | None = None @@ -163,22 +151,6 @@ class Item(PoeModel): alias="utilityMods", ) - # Item properties - properties: list[ItemProperty] | None = None - notable_properties: list[ItemProperty] | None = Field( - default=None, - alias="notableProperties", - ) - requirements: list[ItemProperty] | None = None - additional_properties: list[ItemProperty] | None = Field( - default=None, - alias="additionalProperties", - ) - next_level_requirements: list[ItemProperty] | None = Field( - default=None, - alias="nextLevelRequirements", - ) - # Categories / misc category: dict[str, list[str]] | None = None flavour_text: list[str] | None = Field( @@ -212,8 +184,10 @@ class Item(PoeModel): veiled: bool | None = None foreseeing: bool | None = None - def to_cache(self) -> CacheItem: - return CacheItem(id=self.id, note=self.note) + def to_cache(self, detector: DetectorController) -> CacheItem: + return CacheItem( + id=self.id, note=self.note, category=detector.get_category(self) + ) def __eq__(self, other: CacheItem | object) -> bool: if isinstance(other, CacheItem): @@ -221,6 +195,11 @@ def __eq__(self, other: CacheItem | object) -> bool: return other.id == self.id and other.note == self.note return super().__eq__(other) + def has_price(self) -> bool: + return self.note is not None and re.match( + r"^(~b\/o|~price) [0-9]*[.]?[0-9]+ [^ ]*$", self.note + ) + class Stash(PoeModel): id: str @@ -231,14 +210,19 @@ class Stash(PoeModel): stash_type: str = Field(alias="stashType") league: str | None = None - items: list[Item] = Field(default_factory=list) + items: list[PoeItem] = Field(default_factory=list) - def to_cache(self) -> CacheStash: + def to_cache(self, detector: DetectorController) -> CacheStash: return CacheStash( - league=self.league, items=[item.to_cache() for item in self.items] + league=self.league, items=[item.to_cache(detector) for item in self.items] + ) + + def has_price(self) -> bool: + return self.stash is not None and re.match( + r"^(~b\/o|~price) \d+ [^ ]*$", self.stash ) - def get_item(self, id: str) -> tuple[int, Item | None]: + def get_item(self, id: str) -> tuple[int, PoeItem | None]: for i, item in enumerate(self.items): if item.id == id: return i, item diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py new file mode 100644 index 00000000..561c4d8d --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py @@ -0,0 +1,18 @@ +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeModel, +) + + +class CacheItem(PoeModel): + id: str + note: str + category: str + + +class CacheItemWithContext(CacheItem): + league: str + + +class CacheStash(PoeModel): + league: str + items: list[CacheItem] diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py new file mode 100644 index 00000000..3810a93a --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py @@ -0,0 +1,47 @@ +from pydantic import Field + +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.base import ( + PoeModel, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItemWithContext, +) + + +class OrganizedItems(PoeModel): + removed_items: list[CacheItemWithContext] = Field(default_factory=list) + changed_items: list[PoeItem] = Field(default_factory=list) + new_items: list[PoeItem] = Field(default_factory=list) + + def extend(self, table: "OrganizedItems"): + self.removed_items.extend(table.removed_items) + self.changed_items.extend(table.changed_items) + self.new_items.extend(table.new_items) + + def is_empty(self) -> bool: + return not (self.removed_items or self.changed_items or self.new_items) + + +class OrganizedItemsByCategory(PoeModel): + """ + Field names much match the identifiers of the category detectors + """ + + unique: OrganizedItems = Field(default_factory=OrganizedItems) + unidentified_unique: OrganizedItems = Field(default_factory=OrganizedItems) + + def extend(self, categorized_items: "OrganizedItemsByCategory"): + self.unique.extend(categorized_items.unique) + self.unidentified_unique.extend(categorized_items.unidentified_unique) + + def is_empty(self) -> bool: + for table_name in self.model_fields: + table: OrganizedItems = getattr(self, table_name) + is_empty = table.is_empty() + if not is_empty: + return is_empty + + return True diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py index de6b006e..f28af2de 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py @@ -1,21 +1,35 @@ from abc import ABC, abstractmethod from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( - Item, + PoeItem, Stash, ) -class DetectorBase(ABC): +class SubCategoryDetectorBase(ABC): + wanted_items: list[str] + + def is_wanted(self, item: PoeItem) -> bool: + return item.name in self.wanted_items + + @abstractmethod + def __str__(self): + """Returns a descriptive name for the sub category""" + + +class CategoryDetectorBase(ABC): """ Base class for searching stashes for items we want to store and process further """ + identifier: str + should_cache: bool - wanted_items: list[str] + + detectors: list[SubCategoryDetectorBase] @abstractmethod - def _is_wanted(self, item: Item) -> bool: + def matches_category(self, item: PoeItem) -> bool: """Returns true or false based on the criteria of the detector""" def find_interesting_items( @@ -23,14 +37,21 @@ def find_interesting_items( ): donated_items = list[str]() for i, item in enumerate(donor_stash.items): - if not self._is_wanted(item): + if not self.matches_category(item): continue - if self.should_cache: - receiver_stash.items.append(item) - else: - no_cache_receiver_stash.items.append(item) - donated_items.append(i) + is_wanted = False + for detector in self.detectors: + is_wanted = detector.is_wanted(item) + if is_wanted: + break + + if is_wanted: + if self.should_cache: + receiver_stash.items.append(item) + else: + no_cache_receiver_stash.items.append(item) + donated_items.append(i) for idx in reversed(donated_items): donor_stash.items.pop(idx) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py index cef165b6..95d5778d 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py @@ -4,59 +4,58 @@ from backend_api.app.core.schemas.league import League from pydantic import TypeAdapter +from data_retrieval_app.external_data_retrieval.config import settings from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, + Stash, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( CacheItem, + CacheItemWithContext, CacheStash, - Item, - Stash, ) -from data_retrieval_app.external_data_retrieval.detectors.base import DetectorBase +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.categorized import ( + OrganizedItems, + OrganizedItemsByCategory, +) +from data_retrieval_app.external_data_retrieval.detectors.base import ( + CategoryDetectorBase, +) +from data_retrieval_app.external_data_retrieval.utils import ( + sync_timing_tracker, +) +from data_retrieval_app.logs.logger import data_retrieval_logger as logger class DetectorController: + backend_base_url = settings.BACKEND_BASE_URL + existing_items_url = f"{backend_base_url}/item/existing/" + def __init__( self, - detectors: list[DetectorBase], + detectors: list[CategoryDetectorBase], leagues: list[League], - redis_cache: redis.Redis, ): self.leagues = [league.name for league in leagues] self.detectors = detectors self.local_cache = dict[str, CacheStash]() - self.local_cache[ - "158c93260022f9b39f9e7c6980711ab057c6ef2e28e1cdf00f2c2b5aef2c880c" - ] = CacheStash( - league="Standard", - items=[ - CacheItem( # changed price - id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_updated", - note="~b/o 2 mirror", - ), - CacheItem( # same price - id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402", - note="~b/o 30 chaos", - ), - CacheItem( # removed - id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_removed", - note="~b/o 2 mirror", - ), - ], - ) self.cached_stash_adapter = TypeAdapter(list[CacheStash | None]) - self.redis_cache = redis_cache def _filter_duplicates(self, stashes: list[Stash]) -> list[Stash]: previous_ids = list[str]() non_duplicate_stashes = list[Stash]() - for stash in stashes: + duplicates_found = 0 + for stash in reversed(stashes): if stash.id in previous_ids: + duplicates_found += 1 continue previous_ids.append(stash.id) non_duplicate_stashes.append(stash) + logger.info(f"Filtered away {duplicates_found} duplicate stashes") return non_duplicate_stashes def _filter_never_used( @@ -76,35 +75,38 @@ def _filter_never_used( "unmodifiableExceptChaos", } - def has_price(note: str): - return note is not None and note.startswith(("~b/o", "~price")) - filtered_stashes = list[Stash]() empty_stashes = list[Stash]() + + n_uninteresting_items = 0 + n_items_in = 0 + for stash in stashes: - items: list[Item] = stash.items + items: list[PoeItem] = stash.items filtered_items = [] if not stash.public or stash.league not in self.leagues: # TODO should I already filter leagues? depends on what happens on migration - print(stash.id) continue + n_items_in += len(items) for item in items: if ( uninteresting_item_fields & item.model_dump(exclude_none=True).keys() ): + # if any of these fields are present, it automatically means we are not interested + n_uninteresting_items += 1 continue - if has_price(item.note): + if item.has_price(): pass - elif has_price(stash.stash): + elif stash.has_price(): item.note = stash.stash else: + n_uninteresting_items += 1 continue filtered_items.append(item) - # print(len(filtered_items), stash.id) if filtered_items: stash.items = filtered_items filtered_stashes.append(stash) @@ -112,15 +114,22 @@ def has_price(note: str): stash.items = [] empty_stashes.append(stash) + logger.info( + f"task=filter_never_used n_stashes_in={len(stashes)} n_stashes_found={len(filtered_stashes)} n_items_in={n_items_in} n_items_filtered={n_uninteresting_items}" + ) return filtered_stashes, empty_stashes def _filter_uninteresting_items( self, stashes: list[Stash] - ) -> tuple[list[Stash], list[Item], list[Stash]]: + ) -> tuple[list[Stash], list[PoeItem], list[Stash]]: + n_uninteresting_items = 0 + n_items_in = 0 + interesting_stashes = list[Stash]() - no_cache_new_items = list[Stash]() + no_cache_new_items = list[PoeItem]() filtered_stashes = list[Stash]() for donor_stash in stashes: + n_items_in += len(donor_stash.items) receiver_stash = donor_stash.model_copy(update={"items": []}) no_cache_receiver_stash = donor_stash.model_copy(update={"items": []}) for detector in self.detectors: @@ -131,15 +140,23 @@ def _filter_uninteresting_items( if receiver_stash.items: interesting_stashes.append(receiver_stash) - elif no_cache_receiver_stash.items: + + if no_cache_receiver_stash.items: # items such as unidentified items no_cache_new_items.extend(no_cache_receiver_stash.items) - else: - filtered_stashes.append(donor_stash) + n_uninteresting_items += len(donor_stash.items) + + filtered_stashes.append(donor_stash) + + logger.info( + f"task=filter_uninteresting n_stashes_in={len(stashes)} n_stashes_filtered={len(filtered_stashes)} n_items_in={n_items_in} n_items_filtered={n_uninteresting_items}" + ) return interesting_stashes, no_cache_new_items, filtered_stashes - def _get_cached_stashes(self, ids: list[str]) -> list[CacheStash | None]: + def _get_cached_stashes( + self, redis_cache: redis.Redis, ids: list[str] + ) -> list[CacheStash | None]: """ First checks local cache, then redis cache. This method relies on the local cache being exactly aligned with the redis cache for stashes that exists in both. @@ -148,17 +165,17 @@ def _get_cached_stashes(self, ids: list[str]) -> list[CacheStash | None]: return [] # first local cache - cached_stashes = [] + cached_stashes = list[CacheStash | None]() still_needs_check = list[tuple[int, str]]() for i, id in enumerate(ids): - cached_stash = self.local_cache.get(id) + cached_stash = self.local_cache.get(f"stash:{id}") if cached_stash is None: still_needs_check.append((i, id)) cached_stashes.append(cached_stash) # then redis cache redis_cached_stashes = self.cached_stash_adapter.validate_python( - self.redis_cache.mget([id for _, id in still_needs_check]) + redis_cache.mget([f"stash:{id}" for _, id in still_needs_check]) ) for i, cached_stash in enumerate(redis_cached_stashes): if cached_stash is not None: @@ -167,84 +184,145 @@ def _get_cached_stashes(self, ids: list[str]) -> list[CacheStash | None]: return cached_stashes - def _find_removed_items(self, empty_stash_ids: list[str]) -> list[CacheItem]: + def _find_removed_items( + self, empty_stash_ids: list[str], redis_cache: redis.Redis + ) -> list[CacheItem]: # TODO what if league can change but id stays the same - empty_cached_stashes = self._get_cached_stashes(empty_stash_ids) + empty_cached_stashes = self._get_cached_stashes(redis_cache, empty_stash_ids) removed_items = list[CacheItem]() cached_stashes_to_remove = list[str]() + for id, cached_stash in zip(empty_stash_ids, empty_cached_stashes, strict=True): if cached_stash is not None: - print("cached stash must be removed", id) - cached_stashes_to_remove.append(id) + cached_stashes_to_remove.append(f"stash:{id}") removed_items.extend(cached_stash.items) - [self.local_cache.pop(id, None) for id in cached_stashes_to_remove] - self.redis_cache.delete(cached_stashes_to_remove) + if cached_stashes_to_remove: + [self.local_cache.pop(id, None) for id in cached_stashes_to_remove] + redis_cache.delete(*cached_stashes_to_remove) + logger.info( + f"task=find_removed n_stashes_in={len(empty_stash_ids)} n_stashes_cached={len(cached_stashes_to_remove)} n_removed_items={len(removed_items)}" + ) return removed_items def _find_changed_items_filter_unchanged( - self, stashes: list[Stash] - ) -> tuple[list[Item], list[Item], list[CacheItem]]: + self, stashes: list[Stash], redis_cache: redis.Redis + ) -> tuple[list[CacheItemWithContext], list[PoeItem], list[PoeItem]]: # TODO what if league can change but id stays the same not_empty_stash_ids = [stash.id for stash in stashes] - not_empty_cached_stashes = self._get_cached_stashes(not_empty_stash_ids) + not_empty_cached_stashes = self._get_cached_stashes( + redis_cache, not_empty_stash_ids + ) + + changed_items = list[PoeItem]() + new_items = list[PoeItem]() + removed_items = list[CacheItemWithContext]() - changed_items = list[Item]() - new_items = list[Item]() - removed_items = list[CacheItem]() + n_stashes_cached = 0 + n_items_filtered = 0 stashes_to_cache = dict[str, CacheStash]() for cached_stash, stash in zip(not_empty_cached_stashes, stashes, strict=True): cached_stash_changed = False if cached_stash is not None: + n_stashes_cached += 1 for cached_item in cached_stash.items[:]: idx, new_item = stash.get_item(cached_item.id) if new_item is None: # a cached item doesnt exist anymore - print("cached item no longer exists", cached_item.id) cached_stash_changed = True cached_stash.items.remove(cached_item) - removed_items.append(cached_item) + removed_items.append( + CacheItemWithContext( + id=cached_item.id, + note=cached_item.note, + category=cached_item.category, + league=stash.league, + ) + ) continue if new_item.note == cached_item.note: # Duplicate found in cache - print("duplicate found", new_item.id) + n_items_filtered += 1 stash.items.pop(idx) continue - # Item has changed from cache - print("item has changed", new_item.id) + # PoeItem has changed from cache changed_items.append(new_item) cached_stash_changed = True cached_stash.items.remove(cached_item) - cached_stash.items.extend(stash.to_cache().items) + cached_stash.items.extend(stash.to_cache(self).items) else: # The stash has never been cached cached_stash_changed = True - cached_stash = stash.to_cache() + cached_stash = stash.to_cache(self) new_items.extend(stash.items) if cached_stash_changed and cached_stash.items: - stashes_to_cache[stash.id] = cached_stash + stashes_to_cache[f"stash:{stash.id}"] = cached_stash + + if stashes_to_cache: + self.local_cache.update(stashes_to_cache) + redis_cache.mset(stashes_to_cache) + + logger.info( + f"task=find_changed_filter_unchanged n_stashes_in={len(stashes)} n_stashes_cached={n_stashes_cached} n_items_filtered={n_items_filtered} n_removed_items={len(removed_items)} n_changed_items={len(changed_items)} n_new_items={len(new_items)}" + ) + return removed_items, changed_items, new_items + + def _organize_items( + self, + removed_items: list[CacheItemWithContext], + changed_items: list[PoeItem], + new_items: list[PoeItem], + ) -> OrganizedItemsByCategory: + categorized_items = OrganizedItemsByCategory() + for detector in self.detectors: + table = OrganizedItems() + + for removed_item in removed_items[:]: + if detector.identifier == removed_item.category: + table.removed_items.append(removed_item) + removed_items.remove(removed_item) + + for changed_item in changed_items[:]: + if detector.matches_category(changed_item): + table.changed_items.append(changed_item) + changed_items.remove(changed_item) + + for new_item in new_items[:]: + if detector.matches_category(new_item): + table.new_items.append(new_item) + new_items.remove(new_item) + + setattr(categorized_items, detector.identifier, table) - self.local_cache.update(stashes_to_cache) - self.redis_cache.mset(stashes_to_cache) + return categorized_items - return changed_items, new_items, removed_items + def get_category(self, item: PoeItem) -> str: + for detector in self.detectors: + if detector.matches_category(item): + return detector.identifier + raise ValueError("PoeItem does not match a detector category") + + @sync_timing_tracker def filter_stashes( - self, stashes: list[Stash] - ) -> tuple[list[CacheItem], list[Item], list[Item]]: + self, stashes: list[Stash], redis_cache: redis.Redis + ) -> OrganizedItemsByCategory: """ + Parameters: + :param stashes: a list of stash objects + :param redis_cache: a redis cache Returns: - Removed items: items which need to be closed - Changed items: items which need to start a new entry - New items: items which need to be entered into the system + :Removed items: items which need to be closed + :Changed items: items which need to start a new entry + :New items: items which need to be entered into the system """ stashes = self._filter_duplicates(stashes) @@ -253,33 +331,35 @@ def filter_stashes( ( interesting_stashes, - new_items, + no_cache_new_items, uninteresting_stashes, ) = self._filter_uninteresting_items(filtered_stashes) empty_stash_ids.extend([stash.id for stash in uninteresting_stashes]) - removed_items = self._find_removed_items(empty_stash_ids) + removed_items = self._find_removed_items(empty_stash_ids, redis_cache) ( + more_removed_items, changed_items, more_new_items, - more_removed_items, - ) = self._find_changed_items_filter_unchanged(interesting_stashes) - new_items.extend(more_new_items) + ) = self._find_changed_items_filter_unchanged(interesting_stashes, redis_cache) + + new_items = no_cache_new_items + more_new_items removed_items.extend(more_removed_items) - return removed_items, changed_items, new_items + logger.info( + f"task=total n_removed_items={len(removed_items)} n_changed_items={len(changed_items)} n_new_items={len(new_items)}" + ) + + return self._organize_items(removed_items, changed_items, new_items) if __name__ == "__main__": import json from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( - UniqueArmourDetector, - UniqueJewelDetector, - UniqueJewelleryDetector, - UniqueUnidentifiedDetector, - UniqueWeaponDetector, + UnidentifiedUniqueDetector, + UniqueDetector, ) class RedisCacheDecoy: @@ -318,16 +398,33 @@ def delete(self, keys: list[str]): detector_controller = DetectorController( [ - UniqueArmourDetector(), - UniqueJewelDetector(), - UniqueJewelleryDetector(), - UniqueWeaponDetector(), - UniqueUnidentifiedDetector(), + UniqueDetector(), + UnidentifiedUniqueDetector(), ], [{"name": "Standard"}, {"name": "Hardcore"}, {"name": "Allflame"}], redis_cache=RedisCacheDecoy(), ) + detector_controller.local_cache[ + "158c93260022f9b39f9e7c6980711ab057c6ef2e28e1cdf00f2c2b5aef2c880c" + ] = CacheStash( + league="Standard", + items=[ + CacheItem( # changed price + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_updated", + note="~b/o 2 mirror", + ), + CacheItem( # same price + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402", + note="~b/o 30 chaos", + ), + CacheItem( # removed + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_removed", + note="~b/o 2 mirror", + ), + ], + ) + stashes = json.load(open("stashes.json")) stashes = [ stash diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py index caca95ac..b59d19cb 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py @@ -1,22 +1,91 @@ from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( - Item, + PoeItem, +) +from data_retrieval_app.external_data_retrieval.detectors.base import ( + CategoryDetectorBase, + SubCategoryDetectorBase, ) -from data_retrieval_app.external_data_retrieval.detectors.base import DetectorBase -class UniqueDetector(DetectorBase): +class UniqueJewelDetector(SubCategoryDetectorBase): + wanted_items = [ + "Grand Spectrum", + "Forbidden Flesh", + "Forbidden Flame", + "The Balance of Terror", + "That Which Was Taken", + "Split Personality", + "Thread of Hope", + "The Adorned", + "Impossible Escape", + "Watcher's Eye", + "Bound By Destiny", + "Sublime Vision", + "The Light of Meaning", + "Glorious Vanity", + "Lethal Pride", + "Brutal Restraint", + "Militant Faith", + "Elegant Hubris", + "Voices", + "Heroic Tragedy", + ] + + def __str__(self): + return "Unique Jewel Detector" + + +class UniqueJewelleryDetector(SubCategoryDetectorBase): + wanted_items = [ + "Aul's Uprising", + "Replica Dragonfang's Flight", + "The Utmost", + "Precursor's Emblem", + "Circle of Ambition", + "Circle of Guilt", + "Circle of Anguish", + "Circle of Regret", + "Circle of Fear", + "Circle of Nostalgia", + "Screams of the Desiccated", + ] + + def __str__(self): + return "Unique Jewellery Detector" + + +class UniqueArmourDetector(SubCategoryDetectorBase): + wanted_items = ["Forbidden Shako", "Skin of the Lords", "Shroud of the Lightless"] + + def __str__(self): + return "Unique Armour Detector" + + +class UniqueWeaponDetector(SubCategoryDetectorBase): + wanted_items = ["Paradoxica", "Cane of Kulemak"] + + def __str__(self): + return "Unique Weapon Detector" + + +class UniqueDetector(CategoryDetectorBase): + identifier = "unique" + should_cache = True - def _is_wanted(self, item: Item) -> bool: - if item.rarity is None or item.rarity != "Unique": - return False - if item.name not in self.wanted_items: - return False + detectors = [ + UniqueJewelDetector(), + UniqueJewelleryDetector(), + UniqueArmourDetector(), + UniqueWeaponDetector(), + ] - return True + def matches_category(self, item: PoeItem) -> bool: + return item.identified and item.rarity is not None and item.rarity == "Unique" -class UniqueUnidentifiedDetector(UniqueDetector): +class UnidentifiedUniqueDetector(CategoryDetectorBase): + # TODO separate this into sub categories """ Notes: Precursor's Emblems, Shroud of the Lightless and Paradoxica are not supported. @@ -31,6 +100,8 @@ class UniqueUnidentifiedDetector(UniqueDetector): No, because they are not particularly sought after in their un-id form """ + identifier = "unidentified_unique" + should_cache = False wanted_base_types = [ @@ -89,13 +160,15 @@ class UniqueUnidentifiedDetector(UniqueDetector): "Soulcord.png": "Screams of the Desiccated", } - def _is_wanted(self, item: Item) -> bool: + def matches_category(self, item: PoeItem) -> bool: """ Uses the icon to identify which unique it is, then saving that name inplace. """ - if item.identified: - return False - if item.base_type not in self.wanted_base_types: + if ( + item.identified + or (item.rarity is None or item.rarity != "Unique") + or item.base_type not in self.wanted_base_types + ): return False icon = item.icon.split("/")[-1] @@ -108,64 +181,3 @@ def _is_wanted(self, item: Item) -> bool: def __str__(self): return "Unidentifed Unique detector" - - -class UniqueJewelDetector(UniqueDetector): - wanted_items = [ - "Grand Spectrum", - "Forbidden Flesh", - "Forbidden Flame", - "The Balance of Terror", - "That Which Was Taken", - "Split Personality", - "Thread of Hope", - "The Adorned", - "Impossible Escape", - "Watcher's Eye", - "Bound By Destiny", - "Sublime Vision", - "The Light of Meaning", - "Glorious Vanity", - "Lethal Pride", - "Brutal Restraint", - "Militant Faith", - "Elegant Hubris", - "Voices", - "Heroic Tragedy", - ] - - def __str__(self): - return "Unique Jewel Detector" - - -class UniqueJewelleryDetector(UniqueDetector): - wanted_items = [ - "Aul's Uprising", - "Replica Dragonfang's Flight", - "The Utmost", - "Precursor's Emblem", - "Circle of Ambition", - "Circle of Guilt", - "Circle of Anguish", - "Circle of Regret", - "Circle of Fear", - "Circle of Nostalgia", - "Screams of the Desiccated", - ] - - def __str__(self): - return "Unique Jewellery Detector" - - -class UniqueArmourDetector(UniqueDetector): - wanted_items = ["Forbidden Shako", "Skin of the Lords", "Shroud of the Lightless"] - - def __str__(self): - return "Unique Armour Detector" - - -class UniqueWeaponDetector(UniqueDetector): - wanted_items = ["Paradoxica", "Cane of Kulemak"] - - def __str__(self): - return "Unique Weapon Detector" diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py index aa57a2ab..e58f2be6 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py @@ -6,9 +6,7 @@ ThreadPoolExecutor, wait, ) -from io import StringIO -import pandas as pd import redis from backend_api.app.core.schemas.league import League from pydantic import TypeAdapter @@ -22,8 +20,7 @@ PoEAPIHandler, ) from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_api_data import ( - PoEAPIDataTransformerBase, - UniquePoEAPIDataTransformer, + PoEAPIDataTransformer, ) from data_retrieval_app.external_data_retrieval.utils import ( ProgramTooSlowException, @@ -41,21 +38,14 @@ class ContinuousDataRetrieval: stash_tab_url = "https://api.pathofexile.com/public-stash-tabs" backend_base_url = settings.BACKEND_BASE_URL - modifier_url = f"{backend_base_url}/modifier/" active_league_url = f"{backend_base_url}/league/active_league/" - item_base_type_url = f"{backend_base_url}/itemBaseType/" currency_url = f"{backend_base_url}/currency/" pom_auth_headers = get_superuser_token_headers(backend_base_url) - def __init__( - self, - data_transformers: dict[str, PoEAPIDataTransformerBase], - ): + def __init__(self): self.leagues = self._get_leagues() - self.data_transformers: dict[str, PoEAPIDataTransformerBase] = { - key: data_transformer(self.leagues) - for key, data_transformer in data_transformers.items() - } + + self.data_transformer = PoEAPIDataTransformer(self.leagues) self.poe_api_handler = PoEAPIHandler( url=self.stash_tab_url, @@ -67,35 +57,6 @@ def __init__( url="https://api.poe.watch/exchange/ratios?league={league}&game=poe1" ) - def _get_modifiers(self) -> dict[str, pd.DataFrame]: - response = get_data_safe( - self.modifier_url, headers=self.pom_auth_headers, logger=logger - ) - # Check if the request was successful - modifier_df = pd.DataFrame() - # Load the JSON data into a pandas DataFrame - json_io = StringIO(response.content.decode("utf-8")) - modifier_df = pd.read_json(json_io, dtype=str) - - modifier_types = [ - "implicit", - "explicit", - "delve", - "fractured", - "synthesised", - "unique", - "corrupted", - "enchanted", - "veiled", - ] - modifier_dfs = {} - for modifier_type in modifier_types: - if modifier_type in modifier_df.columns: - modifier_dfs[modifier_type] = modifier_df.loc[ - ~modifier_df[modifier_type].isna() - ] - return modifier_dfs - def _get_leagues(self) -> list[League]: response = get_data_safe( self.active_league_url, headers=self.pom_auth_headers, logger=logger @@ -103,87 +64,29 @@ def _get_leagues(self) -> list[League]: return TypeAdapter(list[League]).validate_python(response.json()) - def _get_item_base_types(self) -> dict[str, int]: - response = get_data_safe( - self.item_base_type_url, headers=self.pom_auth_headers, logger=logger - ) - item_base_type_mapped = {} - item_base_types = [] - - item_base_types = response.json() - if not isinstance(item_base_types, list): - item_base_types = [item_base_types] - for item_base_type in item_base_types: - item_base_type_id = item_base_type["itemBaseTypeId"] - base_type = item_base_type["baseType"] - item_base_type_mapped[base_type] = item_base_type_id - - return item_base_type_mapped - - def _categorize_new_items(self, df: pd.DataFrame) -> dict[str, pd.DataFrame]: - split_dfs = {} - - # TODO not fully exhaustive yet, needs to be updated over time - # category_priority = [ - # "synthesised", - # "fractured", - # "delve", - # "veiled", - # "unique", - # ] - # Needs to take priority, see nebulis and rational doctrine - # not_synth_mask = df["synthesised"].isna() - # split_dfs["synthesised"] = df.loc[~not_synth_mask] - # df = df.loc[not_synth_mask] - - not_unique_mask = df["rarity"] != "Unique" - split_dfs["unique"] = df.loc[~not_unique_mask] - df = df.loc[not_unique_mask] - - # for category in category_priority: - # mask = df[category].isna() - - # split_dfs[category] = df.loc[~mask] - # df = df.loc[mask] - - return split_dfs - def _follow_data_dump_stream(self, cache: redis.Redis): current_hours = find_hours_since_launch(self.leagues) # Only need to refer to one league to see when a new hour starts current_hour = current_hours[self.leagues[0].leagueId] next_hour = current_hour + 1 logger.info("Retrieving modifiers from db.") - modifier_dfs = self._get_modifiers() - item_base_types = self._get_item_base_types() - print(self.leagues) - trade_name_to_currencies = self.currency_api_handler.get_currency_data( + currencies = self.currency_api_handler.get_currency_data( self.leagues, current_hours ) - print(trade_name_to_currencies) - exit() - iter_data = self.poe_api_handler.dump_stream() + self.data_transformer.set_currencies(currencies) + self.data_transformer.set_current_hours(current_hours) + iter_data = self.poe_api_handler.dump_stream(cache) while current_hour < next_hour: - df, next_change_id = next(iter_data) - if df.empty: - continue - split_dfs = self._categorize_new_items(df) - for data_transformer_type in self.data_transformers: - self.data_transformers[data_transformer_type].transform_into_tables( - df=split_dfs[data_transformer_type], - modifier_df=modifier_dfs[data_transformer_type], - # currency_df=currency_df.copy(deep=True), - item_base_types=item_base_types, - current_hours=current_hours, - ) + organized_items, next_change_id = next(iter_data) + self.data_transformer.transform_and_insert(organized_items) if next_change_id is not None: # Only set the next change id once the data has been safely inserted cache.set("next_change_id", next_change_id) current_hours = find_hours_since_launch(self.leagues) - current_hour = current_hour = current_hours[self.leagues[0].leagueId] - for data_transformer_type in self.data_transformers: - self.data_transformers[data_transformer_type].end_of_hour_cleanup() + current_hour = current_hours[self.leagues[0].leagueId] + + self.data_transformer.end_of_hour_cleanup() def retrieve_data(self): logger.info("Program starting up.") @@ -268,11 +171,8 @@ def retrieve_data(self): def main(): logger.info("Starting the program...") setup_logging() - data_transformers = {"unique": UniquePoEAPIDataTransformer} - data_retriever = ContinuousDataRetrieval( - data_transformers=data_transformers, - ) + data_retriever = ContinuousDataRetrieval() data_retriever.retrieve_data() diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py index 92828add..d88b7444 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py @@ -1,201 +1,100 @@ import re - -import pandas as pd - +from collections import defaultdict + +from backend_api.app.core.schemas.item_modifier import ( + ItemModifierCreate, + ItemModifierRoll, +) +from backend_api.app.core.schemas.modifier import GroupedModifier + +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + ItemMod, + PoeItem, +) from data_retrieval_app.logs.logger import transform_logger as logger -pd.set_option("display.max_colwidth", None) - class RollProcessor: - @property - def modifier_df(self) -> pd.DataFrame: - return self._modifier_df - - @modifier_df.setter - def modifier_df(self, modifier_df: pd.DataFrame): - self._modifier_df = modifier_df.drop(["createdAt"], axis=1) - - static_modifier_mask = self._modifier_df["static"] == "True" - self.static_modifier_df = self._modifier_df.loc[static_modifier_mask] - - self.dynamic_modifier_df = self._modifier_df.loc[~static_modifier_mask] - - def add_modifier_df(self, modifier_df: pd.DataFrame): - try: - modifier_df = self.modifier_df - except AttributeError: - self.modifier_df = modifier_df - - def _pre_processing(self, df: pd.DataFrame) -> pd.DataFrame: - no_modifiers_mask = df["modifier"].isna() - df = df.loc[~no_modifiers_mask] - df["modifier"] = df["modifier"].apply(lambda mod: mod["description"]) - df.loc[:, "modifier"] = df[ - "modifier" - ].replace( - r"\\n|\n", " ", regex=True - ) # Replaces newline with a space, so that it does not mess up the regex and matches modifiers in the `modifier` table - # Removes all rows with no modifier (The Adorned) - - return df - - def _process_static( - self, df: pd.DataFrame, static_modifers_mask: pd.Series - ) -> pd.DataFrame: - """ - Static modifiers must be processed first, to reduce the amount of modifiers - processed by the much more expensive dynamic modifier processing. - """ - static_modifier_df = self.static_modifier_df - - static_df = df.loc[static_modifers_mask] - if static_df.empty: - return pd.DataFrame( - columns=static_df.columns.append(static_modifier_df.columns) - ) - static_df.loc[:, "position"] = "0" - static_df.loc[:, "effect"] = static_df.loc[:, "modifier"] - - merged_static_df = static_df.merge( - static_modifier_df, on=["effect", "position"], how="left" - ) - failed_df = merged_static_df.loc[merged_static_df["static"].isna()] - - if not failed_df.empty: - logger.debug( - f"Failed to merge static modifier with modifier in DB.\n{failed_df}" - ) - # remove all modifiers that failed to merge - # NOTE this should never happen - merged_static_df = merged_static_df.loc[~merged_static_df["static"].isna()] - - return merged_static_df - - def _get_rolls(self, dynamic_df: pd.DataFrame) -> pd.DataFrame: - """ - Uses regex matching groups to extract the rolls and adds - the correct effect. - """ - - def extract_rolls(matchobj: re.Match) -> str: - rolls = [ - roll - for roll in matchobj.groups() - if roll not in ["reduced", "increased"] # because of alternate spelling + def __init__(self, modifiers: dict[str, list[GroupedModifier]]): + self.modifiers = modifiers + + self.missing_modifiers = defaultdict[str, set[str]](set) + + def _process_dynamic_modifier( + self, db_mod: GroupedModifier, match: re.Match + ) -> list[ItemModifierRoll]: + position = 0 + rolls = list[ItemModifierRoll]() + for roll in match.groups(): + try: + if roll in ["reduced", "increased"]: + continue + + db_roll = db_mod.rolls[position] + if db_roll.textRolls is not None: + extracted_roll = db_roll.textRolls.index(roll) + else: + extracted_roll = float(roll) + + rolls.append(ItemModifierRoll(position=position, roll=extracted_roll)) + position += 1 + except: + print(roll, position, db_mod) + raise + + return rolls + + def _extract_rolls( + self, modifier: ItemMod, db_modifiers: list[GroupedModifier] + ) -> ItemModifierCreate | None: + # pre processing + effect = modifier.description.replace("\n", " ") + + for db_mod in db_modifiers: + if db_mod.static: + if db_mod.effect == effect: + static_roll = ItemModifierRoll(position=0) + return ItemModifierCreate( + modifierId=db_mod.modifierId, rolls=[static_roll] + ) + + elif (match := db_mod.regex.match(effect)) is not None: + rolls = self._process_dynamic_modifier(db_mod, match) + + return ItemModifierCreate(modifierId=db_mod.modifierId, rolls=rolls) + + return None + + def extract_modifiers(self, item: PoeItem) -> list[ItemModifierCreate] | None: + db_modifiers = self.modifiers[item.name] + + extracted_modifiers = list[ItemModifierCreate]() + if item.name == "The Adorned" and item.explicit_mods is None: + # rare case where the modifier dissapears completely from the item when it rolls 0 + # This can happen with other items, but they will still have some other mods left over + item.explicit_mods = [ + ItemMod( + description=r"0% increased Effect of Jewel Socket Passive Skills containing Corrupted Magic Jewels" + ) ] - - return "matched" + ":-:".join(rolls) - - dynamic_modifier_df = self.dynamic_modifier_df - - # The process must be broken down into a for-loop as the replacement is unique - - dynamic_w_rolls_df = dynamic_df.copy() - for effect, regex in dynamic_modifier_df[["effect", "regex"]].itertuples( - index=False - ): - matched_modifiers = dynamic_df["modifier"].str.replace( - regex, extract_rolls, regex=True, case=False - ) - matched_modifiers_mask = matched_modifiers.str.contains("matched", na=False) - - dynamic_w_rolls_df.loc[matched_modifiers_mask, "effect"] = effect - dynamic_w_rolls_df.loc[ - matched_modifiers_mask, "roll" - ] = matched_modifiers.loc[matched_modifiers_mask] - - dynamic_df.loc[matched_modifiers_mask, "modifier"] = pd.NA - - dynamic_w_rolls_df.loc[:, "roll"] = ( - dynamic_w_rolls_df["roll"].str.removeprefix("matched").str.split(":-:") - ) - del dynamic_df - dynamic_df = dynamic_w_rolls_df - - # If there are rows in the dataframe which contain empty lists, something has failed - failed_df = dynamic_df.loc[dynamic_df["roll"].isna()] - if not failed_df.empty: - logger.critical( - "Failed to add rolls to listed modifiers, this likely means" - " the modifier is legacy or there was a new expansion." - ) - logger.critical( - f"These items have missing modifiers: {failed_df['name'].unique().tolist()}" - ) - logger.critical( - f"These modifiers were not present in the database: {failed_df['effect'].unique().tolist()}" - ) - dynamic_df = dynamic_df.loc[~dynamic_df["roll"].isna()] - - return dynamic_df - - def _process_dynamic( - self, df: pd.DataFrame, static_modifers_mask: pd.Series - ) -> pd.DataFrame: - """ - A much more expensive operation - - Uses the regex column to match incoming modifiers to modifiers in the db. - """ - dynamic_modifier_df = self.dynamic_modifier_df - dynamic_df = df.loc[~static_modifers_mask] # Everything not static is dynamic - if dynamic_df.empty: - return pd.DataFrame( - columns=dynamic_df.columns.append(dynamic_modifier_df.columns) - ) - - dynamic_df.loc[:, "effect"] = dynamic_df.loc[:, "modifier"] - dynamic_df = self._get_rolls(dynamic_df.copy()) - - # Creates a column for position, which contains a list of numerical strings - dynamic_df.loc[:, "position"] = dynamic_df.loc[:, "roll"].apply( - lambda x: [str(i) for i in range(len(x))] - ) - - # Each row describes one roll - dynamic_df = dynamic_df.explode(["roll", "position"]) - - merged_dynamic_df = dynamic_df.merge( - dynamic_modifier_df, on=["effect", "position"], how="left" - ) - - # If all of these fields are still NA, it means that modifier was not matched with a modifier in our DB - failed_df = merged_dynamic_df.loc[merged_dynamic_df["roll"].isna()] - if not failed_df.empty: - logger.exception( - "Some modifiers did not find their counterpart in the database." - " This likely means the modifier is new or has been reworded.\n" - f"{failed_df[['effect', 'roll']].to_string()}" - ) - merged_dynamic_df = merged_dynamic_df.loc[~merged_dynamic_df["roll"].isna()] - - def convert_text_roll_to_index(row: pd.DataFrame) -> int: - text_rolls: str = row["textRolls"] - if text_rolls != "None": - text_rolls = text_rolls.lower().split("|") - roll = text_rolls.index(row["roll"].lower()) + elif item.explicit_mods is None: + logger.critical(f"An item was found with no explicit mods: {item}") + return None + + for mod in item.explicit_mods: + extracted_rolls = self._extract_rolls(mod, db_modifiers) + if extracted_rolls is None: + self.missing_modifiers[item.name].add( + mod.description.replace("\n", " ") + ) else: - roll = row["roll"] - - return roll + extracted_modifiers.append(extracted_rolls) - merged_dynamic_df.loc[:, "roll"] = merged_dynamic_df.apply( - convert_text_roll_to_index, axis=1 - ) # The `roll` column now contains a number + return extracted_modifiers - return merged_dynamic_df - - def add_rolls(self, df: pd.DataFrame) -> pd.DataFrame: - df = self._pre_processing(df.copy()) - - static_modifers_mask = df["modifier"].isin(self.static_modifier_df["effect"]) - - ready_static_df = self._process_static(df.copy(), static_modifers_mask) - ready_dynamic_df = self._process_dynamic(df.copy(), static_modifers_mask) - - processed_df = pd.concat( - (ready_static_df, ready_dynamic_df), axis=0, ignore_index=True - ) # static and dynamic item modifiers are combined into one dataframe again - - return processed_df + def log_missing_modifiers(self): + logger.critical( + "Failed to add rolls to listed modifiers, this likely means" + " the modifier are legacy or there was a new expansion." + f"Missing modifiers: {self.missing_modifiers}" + ) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py index c24fcc57..ebaf86fa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py @@ -1,4 +1,4 @@ -from collections import defaultdict +from collections.abc import Iterator from backend_api.app.core.schemas.currency import ( Currency, @@ -13,7 +13,7 @@ ) from data_retrieval_app.logs.logger import transform_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import post_data_safe +from data_retrieval_app.utils import send_request_safe class TransformCurrencyAPIData: @@ -32,11 +32,11 @@ def _transform( self, exchange_ratios: list[ExchangeRatioItem], current_hours: dict[int, int], - ) -> dict[str, Currency]: + ) -> dict[tuple[int, str], Currency]: """ Since a chaos orb is always worth one chaos orb, ninja does not include it in its price api. """ - trade_name_to_currencies = defaultdict[str, list[Currency]](list) + currencies = dict[tuple[int, str], Currency]() for ratio in exchange_ratios: currency_type = self.name_to_currency.get(ratio.name) if currency_type is None: @@ -61,7 +61,7 @@ def _transform( valueInChaos=value, ) - trade_name_to_currencies[ratio.leagueId].append(currency) + currencies[currency.leagueId, currency.tradeName] = currency for league_id, current_hour in current_hours.items(): name = "Chaos Orb" @@ -76,26 +76,28 @@ def _transform( createdHoursSinceLaunch=current_hour, valueInChaos=1, ) - trade_name_to_currencies[league_id].append(chaos_currency) + currencies[ + chaos_currency.leagueId, chaos_currency.tradeName + ] = chaos_currency - return dict(trade_name_to_currencies) + return currencies - def _insert(self, trade_name_to_currencies: dict[int, list[Currency]]): + def _insert(self, currencies: Iterator[Currency]): prices = list[CurrencyPriceCreate]() - for currencies in trade_name_to_currencies.values(): - for currency in currencies: - prices.append( - CurrencyPriceCreate( - currencyId=currency.currencyId, - leagueId=currency.leagueId, - createdHoursSinceLaunch=currency.createdHoursSinceLaunch, - valueInChaos=currency.valueInChaos, - ) + for currency in currencies: + prices.append( + CurrencyPriceCreate( + currencyId=currency.currencyId, + leagueId=currency.leagueId, + createdHoursSinceLaunch=currency.createdHoursSinceLaunch, + valueInChaos=currency.valueInChaos, ) + ) headers = {"accept": "application/json", "Content-Type": "application/json"} headers.update(self.pom_api_headers) - post_data_safe( + send_request_safe( + "post", self.url, json=TypeAdapter(list[CurrencyPriceCreate]).dump_python(prices), headers=headers, @@ -103,11 +105,11 @@ def _insert(self, trade_name_to_currencies: dict[int, list[Currency]]): def transform_and_insert( self, exchange_ratios: list[ExchangeRatioItem], current_hours: dict[int, int] - ) -> dict[int, list[Currency]]: + ) -> dict[tuple[int, str], Currency]: logger.debug("Transforming exchange ratios into currencies.") - trade_name_to_currencies = self._transform(exchange_ratios, current_hours) + currencies = self._transform(exchange_ratios, current_hours) logger.debug("Inserting currency prices.") - self._insert(trade_name_to_currencies) + self._insert(currencies.values()) logger.debug("Successfully inserted currency data into database.") - return trade_name_to_currencies + return currencies diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py index 8e0b8917..ec89f2e8 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py @@ -1,634 +1,357 @@ -import pandas as pd +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import Literal, TypeVar + +from backend_api.app.core.schemas.currency import Currency +from backend_api.app.core.schemas.item import ( + ItemAvailabilityExpired, + ItemAvailabilityUpdated, + ItemBase, + ItemCreate, + ItemPrice, +) +from backend_api.app.core.schemas.item_base_type import ItemBaseType from backend_api.app.core.schemas.league import League -from requests.exceptions import HTTPError +from backend_api.app.core.schemas.modifier import GroupedModifier +from pydantic import BaseModel, TypeAdapter from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItemWithContext, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.categorized import ( + OrganizedItems, + OrganizedItemsByCategory, +) +from data_retrieval_app.external_data_retrieval.detectors.base import ( + CategoryDetectorBase, +) +from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( + # UnidentifiedUniqueDetector, + UniqueDetector, +) from data_retrieval_app.external_data_retrieval.transforming_data.roll_processor import ( RollProcessor, ) from data_retrieval_app.external_data_retrieval.utils import sync_timing_tracker from data_retrieval_app.logs.logger import transform_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import get_data_safe, insert_data +from data_retrieval_app.utils import send_request_safe -pd.options.mode.chained_assignment = None # default="warn" +class PoEAPIDataTransformer: + url = settings.BACKEND_BASE_URL + pom_auth_headers = get_superuser_token_headers(url) -class PoEAPIDataTransformerBase: def __init__( self, leagues: list[League], ) -> None: logger.debug("Initializing PoEAPIDataTransformer") - self.base_url = settings.BACKEND_BASE_URL - self.pom_auth_headers = get_superuser_token_headers(self.base_url) - self.roll_processor = RollProcessor() + self.pom_auth_headers = get_superuser_token_headers(self.url) + + modifiers = self._get_modifiers() + item_base_types = self._get_item_base_types() + + self.roll_processor = RollProcessor(modifiers) self.league_to_id = {league.name: league.leagueId for league in leagues} logger.debug("Initializing PoEAPIDataTransformer done.") - def _create_item_table(self, df: pd.DataFrame) -> pd.DataFrame: - """ - Creates the basis of the `item` table. - """ - self.item_columns = [ - "itemId", - "id", - "name", - "league", - "baseType", - "typeLine", - "ilvl", - "rarity", - "identified", - "note", - "corrupted", - "delve", - "fractured", - "synthesised", - "replica", - "influences.shaper", - "influences.elder", - "influences.crusader", - "influences.hunter", - "influences.redeemer", - "influences.warlord", - "extended.prefixes", - "extended.suffixes", - "searing", - "tangled", - "foilVariation", - "isRelic", - "stash", + self.transformers: list[TransformerBase] = [ + UniqueTransformer(self.league_to_id, item_base_types, self.roll_processor), + # UnidentifiedUniqueTransformer(self.league_to_id, item_base_types, self.roll_processor), ] - item_df = df.loc[ - :, [column for column in self.item_columns if column in df.columns] - ] # Can't guarantee all columns are present - - return item_df - def _find_not_too_highly_priced_item_mask( - self, - currency_df: pd.DataFrame, - item_currency_merged_df: pd.DataFrame, - ) -> pd.Series: - """ - Some items are too highly priced to be legitimate. A boundary of the price equivelant to - >= 10 mirrors are seen to be too high priced. - - 1. Find currency type and amount equiveland in chaos - """ - - mirror_row = currency_df.loc[currency_df["tradeName"] == "mirror"] - mirror_value = mirror_row["valueInChaos"].get(0, 100_000) - - currency_too_high_mask = ( - item_currency_merged_df["valueInChaos"] - * item_currency_merged_df["currencyAmount"].astype("float") - > mirror_value * 10 + def _get_modifiers(self) -> dict[str, GroupedModifier]: + response = send_request_safe( + "get", + f"{self.url}/modifier/grouped/", + headers=self.pom_auth_headers, + logger=logger, ) + modifiers = TypeAdapter(list[GroupedModifier]).validate_python(response.json()) - return ~currency_too_high_mask - - # @staticmethod - # def extract_price(item_note: str, stash_note: str) -> tuple[float, str] | None: - # """If the item has a price, it returns the price and currency. Otherwise None is returned""" + name_to_modifiers = defaultdict[str, list[GroupedModifier]](list) + for modifier in modifiers: + uniques = modifier.relatedUniques.split("|") + for unique in uniques: + name_to_modifiers[unique].append(modifier) - # def has_price(note: str): - # return note is not None and note.startswith(("~b/o", "~price")) + return dict(name_to_modifiers) - # if has_price(item_note): - # _, price_str, currency = item_note.split(" ") - # elif has_price(stash_note): - # _, price_str, currency = stash_note.split(" ") - # else: - # return None - - # return float(price_str), currency - - @sync_timing_tracker - def _transform_item_table( - self, - item_df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - The `item` table requires a foreign key to the `currency` table. - Everything related to the price of the item is stored in the `note` - attribute. - - There are two types of listings in PoE, exact price and asking price which are - represented by `price` and `b/o` respectively. - """ - - def transform_base_types(element): - return item_base_types[element] - - def get_currency_amount(element): - if len(element) == 3: - return element[1] - return pd.NA - - def get_currency_type(element): - if len(element) == 3: - return element[2] - return "" - - def transform_influences(row: pd.DataFrame, influence_columns: list[str]): - if not row[influence_columns].any(): - return pd.NA - else: - influence_dict = {} - for influence_column in influence_columns: - if row[influence_column]: - influence_dict[ - influence_column.replace("influences.", "") - ] = True - return influence_dict - - item_df["leagueId"] = item_df["league"].map(self.league_to_id) - item_df["createdHoursSinceLaunch"] = item_df["leagueId"].map(current_hours) - - base_type_series = item_df["baseType"] - item_df["itemBaseTypeId"] = base_type_series.apply(transform_base_types) - - influence_columns = [ - column for column in item_df.columns if "influences" in column - ] - item_df["influences"] = item_df.apply( - lambda row: transform_influences(row, influence_columns), axis=1 + def _get_item_base_types(self) -> dict[str, int]: + response = send_request_safe( + "get", + f"{self.url}/itemBaseType/", + headers=self.pom_auth_headers, + logger=logger, ) - rename_extended_map = {} - if "extended.prefixes" in item_df.columns: - rename_extended_map["extended.prefixes"] = "prefixes" - if "extended.suffixes" in item_df.columns: - rename_extended_map["extended.suffixes"] = "suffixes" - - rename_map = {**rename_extended_map, "id": "gameItemId"} - item_df = item_df.rename(columns=rename_map) - - note_series = None - if "note" in item_df.columns: - note_series = item_df["stash"].str.split(" ") - - stash_series = None - if "stash" in item_df.columns: - stash_series = item_df["stash"].str.split(" ") + base_types = TypeAdapter(list[ItemBaseType]).validate_python(response.json()) + base_type_ids = { + base_type.baseType: base_type.itemBaseTypeId for base_type in base_types + } + return base_type_ids - if note_series is not None: - currency_series = note_series - if stash_series is not None: - currency_series = currency_series.where( - item_df["note"].str.contains("~"), stash_series - ) - else: - currency_series = stash_series + def set_current_hours(self, current_hours: dict[int, int]): + for transformer in self.transformers: + transformer.set_current_hours(current_hours) - item_df["currencyAmount"] = currency_series.apply(get_currency_amount) - item_df["currencyType"] = currency_series.apply(get_currency_type) + def set_currencies(self, currencies: dict[tuple[int, str], Currency]): + for transformer in self.transformers: + transformer.set_currencies(currencies) - invalid_amount_mask = ~item_df["currencyAmount"].str.match( - r"^(([0-9]*[.])?[0-9]+)$", na=False - ) - item_df.loc[invalid_amount_mask, "currencyAmount"] = pd.NA - item_df.loc[invalid_amount_mask, "currencyType"] = "" - - item_df = item_df.merge( - currency_df, - how="left", - left_on=["currencyType", "leagueId"], - right_on=["tradeName", "leagueId"], - suffixes=(None, "_y"), - ) + @sync_timing_tracker + def transform_and_insert( + self, + organized_items: OrganizedItemsByCategory, + ): + for transformer in self.transformers: + transformer.transform_and_insert( + getattr(organized_items, transformer.identifier), + ) - price_found_mask = ~item_df["tradeName"].isna() + def end_of_hour_cleanup(self): + self.roll_processor.log_missing_modifiers() + for transformer in self.transformers: + transformer.end_of_hour_cleanup() - self.price_found_mask = price_found_mask - item_df = item_df.loc[self.price_found_mask] +CreateSchemaType = TypeVar("CreateSchemaType", bound=BaseModel) +UpdateSchemaType = TypeVar("UpdateSchemaType", bound=BaseModel) - not_too_high_priced_item_mask = self._find_not_too_highly_priced_item_mask( - currency_df, item_df - ) - self.items_not_too_high_priced_mask = not_too_high_priced_item_mask - - item_df = item_df.loc[self.items_not_too_high_priced_mask] - - return item_df - - @property - def item_table_columns_to_not_drop(self) -> set[str]: - try: - dont_drop_columns = self._item_table_columns_to_not_drop - except AttributeError: - dont_drop_columns = { - "gameItemId", - "name", - "leagueId", - "itemBaseTypeId", - "ilvl", - "rarity", - "identified", - "currencyAmount", - "currencyId", - "corrupted", - "delve", - "fractured", - "synthesised", - "replica", - "influences", - "searing", - "tangled", - "prefixes", - "suffixes", - "foilVariation", - "createdHoursSinceLaunch", - } - self._item_table_columns_to_not_drop = dont_drop_columns - return dont_drop_columns - - @item_table_columns_to_not_drop.setter - def item_table_columns_to_not_drop(self, columns_to_not_drop: set[str]) -> set[str]: - self._item_table_columns_to_not_drop = columns_to_not_drop - - def _update_item_table_columns_to_not_drop(self, *, dont_drop: set[str]) -> None: - columns_to_not_drop = self.item_table_columns_to_not_drop - - self.item_table_columns_to_not_drop = columns_to_not_drop | dont_drop - - def _clean_item_table(self, item_df: pd.DataFrame) -> pd.DataFrame: - """ - Gets rid of unnecessay information, so that only fields needed for the DB remains. - """ - - dont_drop_columns = self.item_table_columns_to_not_drop - - item_df.drop( - item_df.columns.difference(dont_drop_columns), - axis=1, - inplace=True, - errors="ignore", - ) - return item_df +class TransformerBase[RemoveSchemaType, UpdateSchemaType, CreateSchemaType](ABC): + detector: CategoryDetectorBase + identifier: str - def _get_latest_item_id_series(self, item_df: pd.DataFrame) -> pd.Series: - response = get_data_safe( - f"{self.base_url}/item/latest_item_id/", - headers=self.pom_auth_headers, - logger=logger, - ) - latest_item_id = int(response.text) + url = settings.BACKEND_BASE_URL + sub_path: str + url: str - item_id = pd.Series( - range(latest_item_id - len(item_df) + 1, latest_item_id + 1), dtype=int - ) + remove_subpath: str | None = None + update_subpath: str | None = None + insert_subpath: str | None = None - return item_id + pom_auth_headers = get_superuser_token_headers(url) - def _process_item_table( + def __init__( self, - df: pd.DataFrame, - currency_df: pd.DataFrame, + league_to_id: dict[str, int], item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> pd.Series: - """ - Needs to return item ids, as it is used to connect the item modifiers - """ - item_df = self._create_item_table(df) - item_df = self._transform_item_table( - item_df, currency_df, item_base_types, current_hours=current_hours - ) - item_df = self._clean_item_table(item_df) - insert_data( - item_df, - url=self.base_url, - table_name="item", - logger=logger, - headers=self.pom_auth_headers, - ) - item_id = self._get_latest_item_id_series(item_df) - logger.debug("Latest item id found: " + str(item_id)) - return item_id + roll_processor: RollProcessor, + ): + self.identifier = self.detector.identifier + self.url = f"{self.url}/{self.sub_path}/" - @sync_timing_tracker - def _transform_unidentified_item_table( - self, - item_df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - For convenience, all unid items are stored in divine prices - """ - item_df = self._transform_item_table( - item_df, currency_df, item_base_types, current_hours=current_hours - ) + self.type_adapters: dict[Literal["remove", "update", "insert"], TypeAdapter] = { + "remove": TypeAdapter(list[RemoveSchemaType]), + "update": TypeAdapter(list[UpdateSchemaType]), + "insert": TypeAdapter(list[CreateSchemaType]), + } - item_df["chaos_value"] = ( - item_df["currencyAmount"].astype(float) * item_df["valueInChaos"] + self.league_to_id = league_to_id + self.item_base_types = item_base_types + self.roll_processor = roll_processor + + def set_current_hours(self, current_hours: dict[int, int]): + self.current_hours = current_hours + + def set_currencies(self, currencies: dict[tuple[int, str], Currency]): + self.currencies = currencies + self.league_to_mirror = dict[int, Currency]() + for league_id in self.league_to_id.values(): + mirror = currencies[(league_id, "mirror")] + self.league_to_mirror[league_id] = mirror + + @abstractmethod + def _transform_remove( + self, removed_items: list[CacheItemWithContext], current_hours: dict[int, int] + ) -> list[RemoveSchemaType]: + """Transforms items into objects for removing""" + + def _remove(self, removed_items: list[RemoveSchemaType]): + remove_url = self.url + if self.remove_subpath is not None: + remove_url += f"{self.remove_subpath}/" + + send_request_safe( + "patch", + remove_url, + json=self.type_adapters["remove"].validate_python(removed_items), + headers=self.pom_auth_headers, ) - for league in item_df["leagueId"].unique(): - divine_row = currency_df.loc[ - (currency_df["leagueId"] == league) - & (currency_df["tradeName"] == "divine") - ].iloc[0] + def _extract_price(self, item: PoeItem) -> ItemPrice | None: + league_id = self.league_to_id[item.league] - divine_id = divine_row["currencyId"] - divine_value = divine_row["valueInChaos"] + _, currency_amount_str, currency_type_str = item.note.split() - item_league_mask = item_df["leagueId"] == league - item_df.loc[item_league_mask, "currencyId"] = divine_id - item_df.loc[item_league_mask, "currencyAmount"] = ( - item_df.loc[item_league_mask, "chaos_value"] / divine_value - ) - return item_df - - def _clean_unidentified_item_table(self, item_df: pd.DataFrame) -> pd.DataFrame: - """ - Gets rid of unnecessay information, so that only fields needed for the DB remains. - - Only selects unidentified items and always drops suffixes and prefixes - """ - - dont_drop_columns = { - "name", - "itemBaseTypeId", - "createdHoursSinceLaunch", - "leagueId", - "currencyId", - "ilvl", - "currencyAmount", - "identified", - "rarity", - } + currency_type = self.currencies.get((league_id, currency_type_str)) + if currency_type is None: + return None - item_df.drop( - item_df.columns.difference(dont_drop_columns), - axis=1, - inplace=True, - errors="ignore", - ) + currency_amount = float(currency_amount_str) - unidentified_item_df = item_df.loc[~item_df["identified"]] + chaos_price = currency_type.valueInChaos * currency_amount + if self._exceeds_price_threshold(league_id, chaos_price): + return None - return unidentified_item_df - - def _aggregate_unidentified_item_table( - self, - ): - """ - Should run once at the end of every hour. For safety, it also aggregates all previous - hours, in case of previous unfortunate errors. - """ - response = get_data_safe( - f"{self.base_url}/unidentifiedItem/non_aggregated/", - headers=self.pom_auth_headers, - logger=logger, + return ItemPrice( + currencyId=currency_type.currencyId, currencyAmount=currency_amount ) - unid_df = pd.DataFrame(response.json()) - if unid_df.empty: - logger.info("Found no unidentified items to aggregate") - return - - group_cols = [ - "leagueId", - "name", - "itemBaseTypeId", - "createdHoursSinceLaunch", - "ilvl", - "identified", - "currencyId", - "rarity", - ] - g = unid_df.groupby(group_cols) + def _exceeds_price_threshold(self, league_id: int, chaos_price: float) -> bool: + mirror = self.league_to_mirror[league_id] + price_in_mirrors = chaos_price / mirror.valueInChaos - unid_df["calc_avg"] = g["currencyAmount"].transform("mean") - unid_df["calc_std"] = g["currencyAmount"].transform("std") - unid_df["calc_count"] = g["itemId"].transform("count") + return price_in_mirrors > settings.MAX_MIRROR_PRICE - filtered_df = unid_df[ - unid_df["currencyAmount"].between( - unid_df["calc_avg"] - 1.97 * unid_df["calc_std"], - unid_df["calc_avg"] + 1.97 * unid_df["calc_std"], - ) - | unid_df["calc_std"].isna() - ] + @abstractmethod + def _transform_update(self, changed_items: list[PoeItem]) -> list[UpdateSchemaType]: + """Transforms items into objects for updating""" - result_df = filtered_df.groupby(group_cols, as_index=False).agg( - currencyAmount=("currencyAmount", "mean"), - nItems=("calc_count", "first"), - ) - result_df["aggregated"] = True - logger.info("Pushing aggregated unidentified items") - insert_data( - result_df, - url=self.base_url, - table_name="unidentifiedItem/add_aggregated", - logger=logger, + def _update(self, changed_items: list[UpdateSchemaType]): + update_url = self.url + if self.update_subpath is not None: + update_url += f"{self.update_subpath}/" + + send_request_safe( + "put", + update_url, + json=self.type_adapters["update"].dump_python( + changed_items, exclude_none=True + ), headers=self.pom_auth_headers, ) - def _process_unidentified_item_table( - self, - df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> None: - item_df = self._create_item_table(df) - item_df = self._transform_unidentified_item_table( - item_df, currency_df, item_base_types, current_hours=current_hours - ) - item_df = self._clean_unidentified_item_table(item_df) - insert_data( - item_df, - url=self.base_url, - table_name="unidentifiedItem", - logger=logger, + @abstractmethod + def _transform(self, new_items: list[PoeItem]) -> CreateSchemaType: + """Transforms items, using currencies info, into a create schema for the relevant table""" + + def _insert(self, transformed: list[CreateSchemaType]): + insert_url = self.url + if self.insert_subpath is not None: + insert_url += f"{self.insert_subpath}/" + + send_request_safe( + "post", + insert_url, + json=self.type_adapters["insert"].dump_python( + transformed, exclude_none=True + ), headers=self.pom_auth_headers, ) - def _create_item_modifier_table( - self, df: pd.DataFrame, *, item_id: pd.Series - ) -> pd.DataFrame: - """ - The `item_modifier` table heavily relies on what type of item the modifiers - belong to. - """ - raise NotImplementedError("Only available in child classes") + def transform_and_insert(self, items: OrganizedItems): + if items.removed_items: + transformed_remove = self._transform_remove(items.removed_items) + self._remove(transformed_remove) - def _transform_item_modifier_table( - self, - item_modifier_df: pd.DataFrame, - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - The `item_modifier` table heavily relies on what type of item the modifiers - belong to. - """ - raise NotImplementedError("Only available in child classes") - - def _clean_item_modifier_table( - self, item_modifier_df: pd.DataFrame - ) -> pd.DataFrame: - """ - The `item_modifier` table heavily relies on what type of item the modifiers - belong to. - - Gets rid of unnecessay information, so that only fields needed for the DB remains. - """ - raise NotImplementedError("Only available in child classes") - - def _process_item_modifier_table( - self, - df: pd.DataFrame, - item_id: pd.Series, - current_hours: dict[int, int], - ) -> None: - item_modifier_df = self._create_item_modifier_table( - df, item_id=item_id, current_hours=current_hours - ) - item_modifier_df = self._transform_item_modifier_table(item_modifier_df) - item_modifier_df = self._clean_item_modifier_table(item_modifier_df) - insert_data( - item_modifier_df, - url=self.base_url, - table_name="itemModifier", - logger=logger, - headers=self.pom_auth_headers, - ) + if items.changed_items: + transformed_update = self._transform_update(items.changed_items) + self._update(transformed_update) - def transform_into_tables( - self, - df: pd.DataFrame, - modifier_df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> None: - self.roll_processor.add_modifier_df(modifier_df) - try: - logger.debug("Transforming data into tables.") - logger.debug("Processing data tables.") - item_id = self._process_item_table( - df.copy(deep=True), - currency_df=currency_df, - item_base_types=item_base_types, - current_hours=current_hours, - ) - self._process_unidentified_item_table( - df.copy(deep=True), - currency_df=currency_df, - item_base_types=item_base_types, - current_hours=current_hours, + if items.new_items: + transformed = self._transform(items.new_items) + self._insert(transformed) + + @abstractmethod + def end_of_hour_cleanup(self): + """Does necessary cleanup at the end of the hour (eg. unidentified items)""" + + +class UniqueTransformer( + TransformerBase[ItemAvailabilityExpired, ItemAvailabilityUpdated, ItemCreate] +): + detector = UniqueDetector() + sub_path = "item" + + remove_subpath = "availability" + update_subpath = "availability" + + def _transform_remove( + self, removed_items: list[CacheItemWithContext] + ) -> list[ItemAvailabilityExpired]: + transformed = list[ItemAvailabilityExpired]() + for item in removed_items: + league_id = self.league_to_id[item.league] + + transformed.append( + ItemAvailabilityExpired( + gameItemId=item.id, + leagueId=league_id, + validTo=self.current_hours[self.league_to_id], + ) ) - self._process_item_modifier_table( - df.copy(deep=True), - item_id=item_id, - current_hours=current_hours, + + return transformed + + def _transform_update( + self, changed_items: list[PoeItem] + ) -> list[ItemAvailabilityUpdated]: + patches = list[ItemAvailabilityUpdated]() + for item in changed_items: + game_item_id = item.id + league_id = self.league_to_id[item.league] + + price = self._extract_price(item) + if price is None: + continue + + patches.append( + ItemAvailabilityUpdated( + gameItemId=game_item_id, + leagueId=league_id, + price=price, + validFrom=self.current_hours[league_id], + ) ) - logger.debug("Successfully transformed data into tables.") - except HTTPError as e: - logger.exception(f"Something went wrong:\n{repr(e)}") - raise e + return patches - def end_of_hour_cleanup(self): - self._aggregate_unidentified_item_table() + def _transform(self, new_items: list[PoeItem]) -> ItemCreate: + transformed = list[ItemCreate]() + for item in new_items: + game_item_id = item.id + league_id = self.league_to_id[item.league] + price = self._extract_price(item) + if price is None: + continue -class UniquePoEAPIDataTransformer(PoEAPIDataTransformerBase): - @sync_timing_tracker - def _create_item_modifier_table( - self, - df: pd.DataFrame, - *, - item_id: pd.Series, - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - A similiar process to creating the item table, only this time the - relevant column contains a list and not a JSON-object - """ - item_modifier_columns = ["name", "explicitMods", "league"] - item_modifier_df = df.loc[ - self.price_found_mask, - item_modifier_columns, - ] + modifiers = self.roll_processor.extract_modifiers(item) + if modifiers is None: + continue - item_modifier_df = item_modifier_df.loc[ - self.items_not_too_high_priced_mask - ].reset_index() + item_base_type_id = self.item_base_types[item.base_type] - item_modifier_df["itemId"] = item_id - item_modifier_df["leagueId"] = item_modifier_df["league"].map(self.league_to_id) - item_modifier_df["createdHoursSinceLaunch"] = item_modifier_df["leagueId"].map( - current_hours - ) - item_modifier_df = item_modifier_df.explode("explicitMods", ignore_index=True) + first_observed = self.current_hours[league_id] - item_modifier_df.rename({"explicitMods": "modifier"}, axis=1, inplace=True) + item_base = ItemBase( + game_item_id=game_item_id, + league_id=league_id, + item_base_type_id=item_base_type_id, + first_observed=first_observed, + **item.model_dump(), + ) - return item_modifier_df + transformed.append( + ItemCreate(item=item_base, price=price, modifiers=modifiers) + ) + + return transformed + + def end_of_hour_cleanup(self): + pass - @sync_timing_tracker - def _transform_item_modifier_table( - self, - item_modifier_df: pd.DataFrame, - ) -> pd.DataFrame: - item_modifier_df = self.roll_processor.add_rolls(df=item_modifier_df) - - return item_modifier_df - - @property - def item_modifier_table_columns_to_not_drop(self) -> set[str]: - try: - dont_drop_columns = self._item_modifier_table_columns_to_not_drop - except AttributeError: - dont_drop_columns = { - "itemId", - "modifierId", - "position", - "roll", - "createdHoursSinceLaunch", - } - self._item_modifier_table_columns_to_not_drop = dont_drop_columns - return dont_drop_columns - - @item_modifier_table_columns_to_not_drop.setter - def item_modifier_table_columns_to_not_drop( - self, columns_to_not_drop: set[str] - ) -> set[str]: - self._item_modifier_table_columns_to_not_drop = columns_to_not_drop - - def _update_item_modifier_table_columns_to_not_drop( - self, *, dont_drop: set[str] - ) -> None: - columns_to_not_drop = self.item_modifier_table_columns_to_not_drop - self.item_modifier_table_columns_to_not_drop = columns_to_not_drop | dont_drop - - def _clean_item_modifier_table( - self, item_modifier_df: pd.DataFrame - ) -> pd.DataFrame: - """ - Gets rid of unnecessay information, so that only fields needed for the DB remains. - """ - dont_drop_columns = self.item_modifier_table_columns_to_not_drop - - item_modifier_df.drop( - item_modifier_df.columns.difference(dont_drop_columns), - axis=1, - inplace=True, - ) - return item_modifier_df +# class UnidentifiedUniqueTransformer(TransformerBase): +# detector = UnidentifiedUniqueDetector() +# sub_path = "unidentified_item" diff --git a/src/backend_data_retrieval/data_retrieval_app/logs/logger.py b/src/backend_data_retrieval/data_retrieval_app/logs/logger.py index fd5b27f3..956413ca 100644 --- a/src/backend_data_retrieval/data_retrieval_app/logs/logger.py +++ b/src/backend_data_retrieval/data_retrieval_app/logs/logger.py @@ -24,9 +24,12 @@ def setup_logging() -> None: timing_logger = external_data_retrieval_logger.getChild("timing") +data_retrieval_logger = external_data_retrieval_logger.getChild("dataret") + +filter_logger = external_data_retrieval_logger.getChild("filter") + transform_logger = external_data_retrieval_logger.getChild("transform") -data_retrieval_logger = external_data_retrieval_logger.getChild("dataret") data_deposit_logger = main_logger.getChild("datadepo") From 98ce320d7b6a574e8ab0192bd4363e5ab8e80f0c Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 3 Oct 2026 10:34:49 -0230 Subject: [PATCH 15/16] #830 Fixed a bug in the poe api listener where one listener never got to stop after a stop event --- .../data_retrieval/poe_api_handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py index 66618fab..93d2d021 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py @@ -144,7 +144,11 @@ def _follow_stream( reset_event.clear() else: if sent_outgoing: - change_id = incoming.get() + try: + # The timeout is to avoid getting stuck when the other listener has stopped + change_id = incoming.get(timeout=10) + except Empty: + continue sent_outgoing = False From 5f65451745ecf48bc786d39f21f95cc4ec2802c2 Mon Sep 17 00:00:00 2001 From: Bogadisa Date: Sat, 3 Oct 2026 10:37:23 -0230 Subject: [PATCH 16/16] #830 Fixed a few bugs, and started actually acching items. Still requires some testing --- .../app/crud/extensions/crud_item.py | 57 ++++++++++------- .../external_data_retrieval/detectors/base.py | 6 +- .../detectors/detector_controller.py | 61 ++++++++++++------- .../transform_poe_api_data.py | 4 +- 4 files changed, 80 insertions(+), 48 deletions(-) diff --git a/src/backend_api/app/crud/extensions/crud_item.py b/src/backend_api/app/crud/extensions/crud_item.py index a97c810a..cd565dca 100644 --- a/src/backend_api/app/crud/extensions/crud_item.py +++ b/src/backend_api/app/crud/extensions/crud_item.py @@ -180,6 +180,26 @@ async def update_availability( db, incoming=latest_by_item.keys() ) + # removes items not already in the db + incoming_data = list[tuple[int, int, float, int, bool]]() + for item in updated_availability: + item_id = incoming_to_database.get((item.gameItemId, item.leagueId)) + if item_id is None: + continue + + incoming_data.append( + ( + item_id, + item.price.currencyId, + item.price.currencyAmount, + item.validFrom, + item.price.isAsync, + ) + ) + + if not incoming_data: + return [] + incoming_table = values( column("item_id", Integer), column("currency_id", Integer), @@ -187,18 +207,7 @@ async def update_availability( column("valid_from", Integer), column("is_async", Boolean), name="incoming", - ).data( - [ - ( - incoming_to_database[(item.gameItemId, item.leagueId)], - item.price.currencyId, - item.price.currencyAmount, - item.validFrom, - item.price.isAsync, - ) - for item in updated_availability - ] - ) + ).data(incoming_data) # 1. Delete existing same-hour availability db.execute( delete(model_ItemAvailability).where( @@ -258,19 +267,23 @@ async def patch_expired_availability( ], ) + # removes items which don't already exist in db + expired_data = list[tuple[str, int]]() + for item in expired_availability: + item_id = incoming_to_database.get((item.gameItemId, item.leagueId)) + if item_id is None: + continue + + expired_data.append((item_id, item.validTo)) + + if not expired_data: + return [] + expired_table = values( column("item_id", Integer), column("valid_to", Integer), name="expired", - ).data( - [ - ( - incoming_to_database[(item.gameItemId, item.leagueId)], - item.validTo, - ) - for item in expired_availability - ] - ) + ).data(expired_data) # 1. Delete existing same-hour availability db.execute( delete(model_ItemAvailability).where( @@ -288,7 +301,7 @@ async def patch_expired_availability( model_ItemAvailability.validTo.is_(None), ) .values( - validTo=expired_table.c.valid_from, + validTo=expired_table.c.valid_to, ) ) .scalars() diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py index f28af2de..75d97afa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py @@ -34,8 +34,8 @@ def matches_category(self, item: PoeItem) -> bool: def find_interesting_items( self, donor_stash: Stash, receiver_stash: Stash, no_cache_receiver_stash: Stash - ): - donated_items = list[str]() + ) -> bool: + donated_items = list[int]() for i, item in enumerate(donor_stash.items): if not self.matches_category(item): continue @@ -55,3 +55,5 @@ def find_interesting_items( for idx in reversed(donated_items): donor_stash.items.pop(idx) + + return len(donated_items) > 0 diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py index 95d5778d..9d5370dd 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py @@ -1,3 +1,4 @@ +import json from typing import Any import redis @@ -41,7 +42,7 @@ def __init__( self.detectors = detectors self.local_cache = dict[str, CacheStash]() - self.cached_stash_adapter = TypeAdapter(list[CacheStash | None]) + self.cached_stash_adapter = TypeAdapter(CacheStash) def _filter_duplicates(self, stashes: list[Stash]) -> list[Stash]: previous_ids = list[str]() @@ -55,7 +56,9 @@ def _filter_duplicates(self, stashes: list[Stash]) -> list[Stash]: previous_ids.append(stash.id) non_duplicate_stashes.append(stash) - logger.info(f"Filtered away {duplicates_found} duplicate stashes") + logger.info( + f"task=filter_duplicates n_stashes_in={len(stashes)} n_duplicate_stashes={duplicates_found}" + ) return non_duplicate_stashes def _filter_never_used( @@ -132,22 +135,27 @@ def _filter_uninteresting_items( n_items_in += len(donor_stash.items) receiver_stash = donor_stash.model_copy(update={"items": []}) no_cache_receiver_stash = donor_stash.model_copy(update={"items": []}) + found_items = False for detector in self.detectors: # the donor stash is modified inplace, with its items moving over to the receiver stashes inplace - detector.find_interesting_items( - donor_stash, receiver_stash, no_cache_receiver_stash + found_items = ( + detector.find_interesting_items( + donor_stash, receiver_stash, no_cache_receiver_stash + ) + or found_items ) - if receiver_stash.items: - interesting_stashes.append(receiver_stash) - - if no_cache_receiver_stash.items: - # items such as unidentified items - no_cache_new_items.extend(no_cache_receiver_stash.items) + if found_items: + if receiver_stash.items: + interesting_stashes.append(receiver_stash) - n_uninteresting_items += len(donor_stash.items) + if no_cache_receiver_stash.items: + # items such as unidentified items + no_cache_new_items.extend(no_cache_receiver_stash.items) + else: + n_uninteresting_items += len(donor_stash.items) - filtered_stashes.append(donor_stash) + filtered_stashes.append(donor_stash) logger.info( f"task=filter_uninteresting n_stashes_in={len(stashes)} n_stashes_filtered={len(filtered_stashes)} n_items_in={n_items_in} n_items_filtered={n_uninteresting_items}" @@ -171,31 +179,42 @@ def _get_cached_stashes( cached_stash = self.local_cache.get(f"stash:{id}") if cached_stash is None: still_needs_check.append((i, id)) - cached_stashes.append(cached_stash) + cached_stashes.append(None) + else: + cached_stashes.append( + self.cached_stash_adapter.validate_json(cached_stash) + ) # then redis cache - redis_cached_stashes = self.cached_stash_adapter.validate_python( - redis_cache.mget([f"stash:{id}" for _, id in still_needs_check]) + redis_cached_stashes = redis_cache.mget( + [f"stash:{id}" for _, id in still_needs_check] ) for i, cached_stash in enumerate(redis_cached_stashes): if cached_stash is not None: idx, _ = still_needs_check[i] - cached_stashes[idx] = cached_stash + cached_stashes[idx] = self.cached_stash_adapter.validate_json( + cached_stash + ) return cached_stashes def _find_removed_items( self, empty_stash_ids: list[str], redis_cache: redis.Redis - ) -> list[CacheItem]: + ) -> list[CacheItemWithContext]: # TODO what if league can change but id stays the same empty_cached_stashes = self._get_cached_stashes(redis_cache, empty_stash_ids) - removed_items = list[CacheItem]() + removed_items = list[CacheItemWithContext]() cached_stashes_to_remove = list[str]() for id, cached_stash in zip(empty_stash_ids, empty_cached_stashes, strict=True): if cached_stash is not None: cached_stashes_to_remove.append(f"stash:{id}") - removed_items.extend(cached_stash.items) + for cached_item in cached_stash.items: + removed_items.append( + CacheItemWithContext( + **cached_item.model_dump(), league=cached_stash.league + ) + ) if cached_stashes_to_remove: [self.local_cache.pop(id, None) for id in cached_stashes_to_remove] @@ -264,7 +283,7 @@ def _find_changed_items_filter_unchanged( new_items.extend(stash.items) if cached_stash_changed and cached_stash.items: - stashes_to_cache[f"stash:{stash.id}"] = cached_stash + stashes_to_cache[f"stash:{stash.id}"] = cached_stash.model_dump_json() if stashes_to_cache: self.local_cache.update(stashes_to_cache) @@ -355,8 +374,6 @@ def filter_stashes( if __name__ == "__main__": - import json - from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( UnidentifiedUniqueDetector, UniqueDetector, diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py index ec89f2e8..2931c6b8 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py @@ -184,7 +184,7 @@ def _remove(self, removed_items: list[RemoveSchemaType]): send_request_safe( "patch", remove_url, - json=self.type_adapters["remove"].validate_python(removed_items), + json=self.type_adapters["remove"].dump_python(removed_items), headers=self.pom_auth_headers, ) @@ -287,7 +287,7 @@ def _transform_remove( ItemAvailabilityExpired( gameItemId=item.id, leagueId=league_id, - validTo=self.current_hours[self.league_to_id], + validTo=self.current_hours[league_id] + 1, ) )