diff --git a/src/.env b/src/.env index a78e29dc..42f9365c 100644 --- a/src/.env +++ b/src/.env @@ -38,8 +38,8 @@ TURNSTILE_SECRET_KEY=1x0000000000000000000000000000000AA # Data retrieval MANUAL_NEXT_CHANGE_ID=True -# Change ID Retrieved 28.06.2026: -NEXT_CHANGE_ID=3176140940-3103957281-3029294684-3370770619-3261964453 +# Change ID Retrieved 13.09.2026: +NEXT_CHANGE_ID=3273127146-3199406693-3124831624-3474177832-3363107859 POE_PUBLIC_STASHES_AUTH_TOKEN=changethis # Enables the public stashes creation script in data_ret/app/tests/scripts. # Set value "True" to activate: @@ -68,9 +68,7 @@ OATH_ACC_TOKEN_CONTACT_EMAIL=changethis # Redis Cache REDIS_PORT=6379 REDIS_SERVER=cache -REDIS_CACHE=0 REDIS_PASSWORD=changethis -REDIS_URI=redis://:${REDIS_PASSWORD}@${REDIS_SERVER}:${REDIS_PORT}/${REDIS_CACHE} # Vector VECTOR_PORT=8686 diff --git a/src/backend_api/Dockerfile b/src/backend_api/Dockerfile index 6527843d..34373b5f 100644 --- a/src/backend_api/Dockerfile +++ b/src/backend_api/Dockerfile @@ -33,7 +33,7 @@ COPY ./scripts /app/scripts COPY ./pyproject.toml ./uv.lock ./alembic.ini /app/ -COPY ./app /app/app +COPY ./app /app/backend_api/app # Sync the project # Ref: https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers diff --git a/src/backend_api/alembic.ini b/src/backend_api/alembic.ini index 24841c2b..93864b35 100644 --- a/src/backend_api/alembic.ini +++ b/src/backend_api/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = app/alembic +script_location = backend_api/app/alembic # template used to generate migration files # file_template = %%(rev)s_%%(slug)s diff --git a/src/backend_api/app/alembic/env.py b/src/backend_api/app/alembic/env.py index fff12f53..9a514cfc 100644 --- a/src/backend_api/app/alembic/env.py +++ b/src/backend_api/app/alembic/env.py @@ -6,8 +6,8 @@ import os from alembic import context -from app.core.models.models import Base -from app.core.config import settings +from backend_api.app.core.models.models import Base +from backend_api.app.core.config import settings def get_url(): diff --git a/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py b/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py index 8fc6246d..18a82acf 100644 --- a/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py +++ b/src/backend_api/app/alembic/versions/17daa1c96438_removing_modifier_auto_increment.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "17daa1c96438" diff --git a/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py new file mode 100644 index 00000000..36810f3d --- /dev/null +++ b/src/backend_api/app/alembic/versions/23d7asdb13df_split_modifier_table.py @@ -0,0 +1,178 @@ +"""optimize item table + +Revision ID: 23d7asdb13df +Revises: dd4abb2b6541 +Create Date: 2026-07-23 15:20:18.565656 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger + +# revision identifiers, used by Alembic. +revision: str = "23d7asdb13df" +down_revision: Union[str, None] = "dd4abb2b6541" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +modifier_id_trigger = ReplaceableTrigger( + "increment_modifier_id", + "modifier", + """ + RETURNS trigger AS ${name}$ + DECLARE + exists boolean; + BEGIN + exists := EXISTS(SELECT 1 FROM modifier WHERE "effect" = NEW.effect); + IF NOT exists THEN + NEW."modifierId" := nextval('modifier_id_seq'); + + ELSIF exists THEN + NEW."modifierId" := (SELECT "modifierId" FROM modifier WHERE "effect" = NEW.effect LIMIT 1); + END IF; + + RETURN NEW; + END; + ${name}$ LANGUAGE plpgsql; + """, + """ + BEFORE INSERT ON {table} + FOR EACH ROW + EXECUTE FUNCTION {name}(); + """, +) + + +def upgrade() -> None: + op.drop_trigger(modifier_id_trigger) + + op.execute(""" + DROP SEQUENCE modifier_id_seq; + """) + + op.create_table( + "modifier_roll", + sa.Column("modifierId", sa.SmallInteger(), nullable=False), + sa.Column("position", sa.SmallInteger(), nullable=False), + sa.Column("minRoll", sa.Float(4)), + sa.Column("maxRoll", sa.Float(4)), + sa.Column("textRolls", sa.ARRAY(sa.Text())), + sa.PrimaryKeyConstraint("modifierId", "position"), + sa.CheckConstraint( + """ modifier_roll."maxRoll" >= modifier_roll."minRoll" """, + name="check_modifier_maxRoll_greaterThan_minRoll", + ), + ) + + op.execute(""" + INSERT INTO modifier_roll + SELECT + m."modifierId", + m.position, + m."minRoll", + m."maxRoll", + string_to_array(m."textRolls", '|') + FROM modifier m + """) + + with op.batch_alter_table("item_modifier") as batch_op: + batch_op.drop_constraint("fk_item_modifier_modifier", type_="foreignkey") + batch_op.create_foreign_key( + "fk_item_modifier_modifier_roll", + "modifier_roll", + ["modifierId", "position"], + ["modifierId", "position"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + + with op.batch_alter_table("modifier") as batch_op: + batch_op.drop_constraint("modifier_pkey", type_="primary") + batch_op.execute(""" + DELETE FROM modifier m + WHERE m.position > 0 + """) + batch_op.drop_column("position") + batch_op.drop_column("minRoll") + batch_op.drop_column("maxRoll") + batch_op.drop_column("textRolls") + batch_op.create_primary_key("pk_modifier", ["modifierId"]) + batch_op.create_unique_constraint("uq_effect", ["effect"]) + + with op.batch_alter_table("modifier_roll") as batch_op: + batch_op.create_foreign_key( + "fk_modifier_roll_modifier", + "modifier", + ["modifierId"], + ["modifierId"], + ondelete="CASCADE", + onupdate="CASCADE", + ) + + +def downgrade() -> None: + # Remove the FK from modifier_roll -> modifier first. + with op.batch_alter_table("modifier_roll") as batch_op: + batch_op.drop_constraint( + "fk_modifier_roll_modifier", + type_="foreignkey", + ) + + with op.batch_alter_table("modifier") as batch_op: + batch_op.drop_constraint("uq_effect") + batch_op.add_column(sa.Column("position", sa.SmallInteger())) + batch_op.add_column(sa.Column("minRoll", sa.Float(4))) + batch_op.add_column(sa.Column("maxRoll", sa.Float(4))) + batch_op.add_column(sa.Column("textRolls", sa.Text())) + + batch_op.create_primary_key( + "modifier_pkey", + ["modifierId", "position"], + ) + + # Restore the rows that were deleted during upgrade and populate + # the position 0 rows with the data from modifier_roll. + op.execute(""" + INSERT INTO modifier ( + "modifierId", + position, + "minRoll", + "maxRoll", + "textRolls" + ) + SELECT + mr."modifierId", + mr.position, + mr."minRoll", + mr."maxRoll", + array_to_string(mr."textRolls", '|') + FROM modifier_roll mr + """) + + # Restore the original FK from item_modifier -> modifier. + with op.batch_alter_table("item_modifier") as batch_op: + batch_op.drop_constraint( + "fk_item_modifier_modifier_roll", + type_="foreignkey", + ) + batch_op.create_foreign_key( + "fk_item_modifier_modifier", + "modifier", + ["modifierId"], + ["modifierId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + + # modifier_roll is no longer needed. + op.drop_table("modifier_roll") + + op.execute("CREATE SEQUENCE modifier_id_seq;") + op.execute( + """SELECT setval('modifier_id_seq', (SELECT MAX("modifierId") FROM modifier));""" + ) + op.create_trigger(modifier_id_trigger) diff --git a/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py b/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py index 7586677d..49a9125f 100644 --- a/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py +++ b/src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "965e766db0a0" diff --git a/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py new file mode 100644 index 00000000..a78fbab3 --- /dev/null +++ b/src/backend_api/app/alembic/versions/dd4abb2b6541_optimize_item_table.py @@ -0,0 +1,516 @@ +"""optimize item table + +Revision ID: dd4abb2b6541 +Revises: ee1239y6yfda +Create Date: 2026-07-23 15:20:18.565656 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "dd4abb2b6541" +down_revision: Union[str, None] = "ee1239y6yfda" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "_temp_item_availability", + sa.Column("temp_game_item_id", sa.Text(), nullable=False), # Temporary column + sa.Column( + "temp_league_id", sa.SmallInteger(), nullable=False + ), # Temporary column + # sa.Column("itemId", sa.Integer(), nullable=False), # Will be added later + sa.Column("currencyId", sa.SmallInteger(), nullable=False), + sa.Column("currencyAmount", sa.Float(4), nullable=False), + sa.Column("isAsync", sa.Boolean()), + sa.Column("validFrom", sa.SmallInteger(), nullable=False), + sa.Column("validTo", sa.SmallInteger()), + sa.ForeignKeyConstraint( + ["currencyId"], + ["currency_type.currencyId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ), + ) + + op.execute(""" + INSERT INTO _temp_item_availability ( + temp_game_item_id, + temp_league_id, + "currencyId", + "currencyAmount", + "validFrom", + "validTo" + ) + WITH ordered AS ( + SELECT + i.*, + LAG(i."createdHoursSinceLaunch") OVER w AS prev_hour, + LAG(i."leagueId") OVER w AS prev_league, + LAG(i."currencyId") OVER w AS prev_currency, + LAG(i."currencyAmount") OVER w AS prev_amount + FROM item i + WINDOW w AS ( + PARTITION BY i."gameItemId" + ORDER BY i."createdHoursSinceLaunch" + ) + ), + marked AS ( + SELECT + *, + CASE + WHEN + prev_hour IS NULL + OR o."createdHoursSinceLaunch" > prev_hour + 1 + OR o."leagueId" IS DISTINCT FROM prev_league + OR o."currencyId" IS DISTINCT FROM prev_currency + OR o."currencyAmount" IS DISTINCT FROM prev_amount + THEN 1 + ELSE 0 + END AS new_listing + FROM ordered o + ), + grouped AS ( + SELECT + *, + SUM(new_listing) OVER ( + PARTITION BY m."gameItemId" + ORDER BY m."createdHoursSinceLaunch" + ) AS grp + FROM marked m + ), + collapsed AS ( + SELECT + g."gameItemId", + g."leagueId", + g."currencyId", + g."currencyAmount", + grp, + MIN(g."createdHoursSinceLaunch") AS "validFrom", + MAX(g."createdHoursSinceLaunch") + 1 AS observedValidTo + FROM grouped g + GROUP BY + g."gameItemId", + g."leagueId", + g."currencyId", + g."currencyAmount", + grp + ) + SELECT + c."gameItemId", + c."leagueId", + c."currencyId", + c."currencyAmount", + c."validFrom", + CASE + WHEN LEAD(grp) OVER ( + PARTITION BY c."gameItemId" + ORDER BY grp + ) IS NULL + THEN c."validFrom" + 1 + ELSE observedValidTo + END AS "validTo" + + FROM collapsed c; + """) + # TODO optimize when writing the plotting query + # sa.Index("ix_item_availability_itemId_validFrom", "itemId", "validFrom"), + + with op.batch_alter_table("item") as batch_op: + batch_op.drop_column("prefixes") + batch_op.drop_column("suffixes") + batch_op.drop_column("currencyAmount") + batch_op.drop_column("currencyId") + batch_op.execute(""" + DELETE FROM item i + WHERE i."itemId" NOT IN ( + SELECT MIN(i2."itemId") + FROM item i2 + GROUP BY i2."gameItemId", i2."leagueId" + ) + """) + batch_op.execute(""" + DELETE FROM item i + WHERE i."itemId" NOT IN ( + SELECT im."itemId" + FROM item_modifier im + ) + """) + batch_op.alter_column("itemId", new_column_name="old_item_id") + + # Rebuilds the item table to remove the hypertable. + op.execute(""" + CREATE TABLE _item_temp AS + SELECT DISTINCT ON ("gameItemId", "leagueId") + ROW_NUMBER() OVER (ORDER BY "gameItemId", "leagueId") AS "itemId", + * + FROM item i + ORDER BY + "gameItemId", + "leagueId"; + + DROP TABLE item; + ALTER TABLE _item_temp + ALTER COLUMN "itemId" SET NOT NULL; + ALTER TABLE _item_temp + ALTER COLUMN "itemId" ADD GENERATED ALWAYS AS IDENTITY; + SELECT setval( + pg_get_serial_sequence('_item_temp', 'itemId'), + COALESCE((SELECT MAX("itemId") FROM _item_temp), 1), + TRUE + ); + + ALTER TABLE _item_temp RENAME TO item; + """) + + with op.batch_alter_table("item") as batch_op: + batch_op.drop_column("delve") + batch_op.drop_column("foilVariation") + batch_op.create_primary_key("item_pkey", ["itemId"]) + batch_op.create_foreign_key( + "fk_item_league", + "league", + ["leagueId"], + ["leagueId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + batch_op.create_foreign_key( + "fk_item_item_base_type", + "item_base_type", + ["itemBaseTypeId"], + ["itemBaseTypeId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + batch_op.alter_column("gameItemId", nullable=False) + batch_op.alter_column("leagueId", nullable=False) + batch_op.alter_column( + "createdHoursSinceLaunch", new_column_name="firstObserved", nullable=False + ) + batch_op.alter_column("name", nullable=False) + batch_op.alter_column("itemBaseTypeId", nullable=False) + batch_op.alter_column("rarity", nullable=False) + batch_op.alter_column("ilvl", nullable=False) + batch_op.alter_column("identified", nullable=False) + + # TODO update when writing the plotting query + batch_op.create_index( + "ix_item_leagueId_itemBaseTypeId", + ["leagueId", "itemBaseTypeId", "firstObserved"], + ) + + batch_op.create_unique_constraint( + "uq_item_game_item_id_league_id", ["gameItemId", "leagueId"] + ) + + op.create_table( + "item_availability", + sa.Column( + "availabilityId", + sa.Integer(), + sa.Identity(always=False, start=1, increment=1), + nullable=False, + primary_key=True, + ), + sa.Column("itemId", sa.Integer(), nullable=False), + sa.Column("currencyId", sa.SmallInteger(), nullable=False), + sa.Column("currencyAmount", sa.Float(4), nullable=False), + sa.Column("isAsync", sa.Boolean()), + sa.Column("validFrom", sa.SmallInteger(), nullable=False), + sa.Column("validTo", sa.SmallInteger()), + sa.ForeignKeyConstraint( + ["currencyId"], + ["currency_type.currencyId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ), + sa.ForeignKeyConstraint( + ["itemId"], + ["item.itemId"], + ondelete="CASCADE", + onupdate="CASCADE", + ), + # TODO optimize when writing the plotting query + sa.Index("ix_item_availability_itemId_validFrom", "itemId", "validFrom"), + sa.UniqueConstraint("itemId", "validFrom"), + sa.CheckConstraint( + """ + item_availability."validTo" IS NULL + OR item_availability."validTo" > item_availability."validFrom" + """, + name="check_positive_duration", + ), + ) + op.execute(""" + INSERT INTO item_availability ( + "itemId", + "currencyId", + "currencyAmount", + "validFrom", + "validTo" + ) + SELECT + i."itemId", + ia."currencyId", + ia."currencyAmount", + ia."validFrom", + ia."validTo" + FROM item i + LEFT JOIN _temp_item_availability ia + ON i."gameItemId" = ia.temp_game_item_id AND i."leagueId" = ia.temp_league_id; + """) + + op.drop_table("_temp_item_availability") + + # Rebuilds the item_modifier table to use the new itemId, and to remove the hypertable. + op.execute(""" + CREATE TABLE _item_modifier_temp AS + SELECT + i."itemId"::INT, + im."modifierId", + im.position, + ROW_NUMBER() OVER ( + PARTITION BY i."itemId", im."modifierId", im.position + )::SMALLINT AS instance, + im.roll + FROM item i + LEFT JOIN item_modifier im + ON i.old_item_id = im."itemId"; + + DROP TABLE item_modifier; + + ALTER TABLE _item_modifier_temp RENAME TO item_modifier; + """) + op.drop_column("item", "old_item_id") + + with op.batch_alter_table("item_modifier") as batch_op: + batch_op.alter_column("itemId", nullable=False) + batch_op.alter_column("modifierId", nullable=False) + batch_op.alter_column("position", nullable=False) + batch_op.alter_column("instance", nullable=False) + batch_op.alter_column( + "roll", existing_type=sa.Float(), type_=sa.Float(4), nullable=True + ) + batch_op.create_primary_key( + "item_modifier_pkey", ["itemId", "modifierId", "position", "instance"] + ) + batch_op.create_foreign_key( + "fk_item_modifier_item", + "item", + ["itemId"], + ["itemId"], + ondelete="CASCADE", + onupdate="CASCADE", + ) + batch_op.create_foreign_key( + "fk_item_modifier_modifier", + "modifier", + ["modifierId", "position"], + ["modifierId", "position"], + ondelete="RESTRICT", + onupdate="CASCADE", + ) + # batch_op.create_index( + # "ix_item_modifier_modifierId_itemId", + # ["modifierId", "itemId"], + # unique=False, + # ) + + +def downgrade() -> None: + raise NotImplementedError("Downgrade is not supported for this migration") + # Remove the FK before restoring the old item schema. + # op.drop_constraint( + # "fk_item_availability_item", + # "item_availability", + # type_="foreignkey", + # ) + # # Restore the columns that were removed by upgrade(). + # with op.batch_alter_table("item") as batch_op: + # batch_op.drop_constraint( + # "item_pkey", + # type_="primary", + # ) + + # batch_op.alter_column( + # "itemId", + # new_column_name="gameItemId", + # ) + + # batch_op.add_column( + # sa.Column( + # "itemId", + # sa.Integer(), + # sa.Identity(always=False, start=1, increment=1), + # nullable=False, + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "prefixes", + # sa.Integer(), + # nullable=False, + # server_default="0", + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "suffixes", + # sa.Integer(), + # nullable=False, + # server_default="0", + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "currencyId", + # sa.Integer(), + # nullable=True, + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "currencyAmount", + # sa.Float(4), + # nullable=True, + # ) + # ) + + # batch_op.add_column( + # sa.Column( + # "createdHoursSinceLaunch", + # sa.Integer(), + # nullable=True, + # ) + # ) # Build the complete historical item set from item_availability. + # # + # # The upgraded table still contains one representative item row for + # # every (gameItemId, leagueId). We use that row as the template for + # # all of the columns that were not part of item_availability. + # # + # # validFrom is inclusive and validTo is exclusive. + # op.execute(""" + # CREATE TEMP TABLE _item_template + # ON COMMIT DROP + # AS + # SELECT DISTINCT ON ("gameItemId", "leagueId") + # "gameItemId", + # "foilVariation", + # "identified", + # "corrupted", + # "delve", + # "fractured", + # "synthesised", + # "replica", + # "searing", + # "tangled", + # "influences", + # "name", + # "itemBaseTypeId", + # "ilvl", + # "rarity", + # "leagueId" + # FROM item + # ORDER BY + # "gameItemId", + # "leagueId", + # "itemId"; + # """) + + # # Remove the representative rows created by the upgrade. They are + # # replaced by the reconstructed hourly history below. + # # + # # No metadata is lost because it has already been copied into the + # # temporary template table. + # op.execute(""" + # DELETE FROM item; + # """) + + # # Reconstruct one row per hour. + # # + # # item_availability contains intervals: + # # + # # [validFrom, validTo) + # # + # # so generate_series(validFrom, validTo - 1) gives exactly the + # # original hourly rows represented by each interval. + # # + # # A new itemId is generated by the identity column. + # op.execute(""" + # INSERT INTO item ( + # "gameItemId", + # "prefixes", + # "suffixes", + # "foilVariation", + # "identified", + # "corrupted", + # "delve", + # "fractured", + # "synthesised", + # "replica", + # "searing", + # "tangled", + # "influences", + # "name", + # "itemBaseTypeId", + # "createdHoursSinceLaunch", + # "leagueId", + # "currencyId", + # "ilvl", + # "currencyAmount", + # "rarity" + # ) + # SELECT + # ia."itemId" AS "gameItemId", + # 0 AS "prefixes", + # 0 AS "suffixes", + # t."foilVariation", + # t."identified", + # t."corrupted", + # t."delve", + # t."fractured", + # t."synthesised", + # t."replica", + # t."searing", + # t."tangled", + # t."influences", + # t."name", + # t."itemBaseTypeId", + # h."createdHoursSinceLaunch", + # ia."leagueId", + # ia."currencyId", + # t."ilvl", + # ia."currencyAmount", + # t."rarity" + # FROM item_availability ia + # JOIN _item_template t + # ON t."gameItemId" = ia."itemId" + # AND t."leagueId" = ia."leagueId" + # CROSS JOIN LATERAL generate_series( + # ia."validFrom", + # ia."validTo" - 1 + # ) AS h("createdHoursSinceLaunch") + # ORDER BY + # ia."itemId", + # ia."leagueId", + # h."createdHoursSinceLaunch"; + # """) + + # # Restore the original identity primary key. + # with op.batch_alter_table("item") as batch_op: + # batch_op.create_primary_key( + # "item_pkey", + # ["itemId"], + # ) + # op.drop_table("item_availability") diff --git a/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py b/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py index 47942eee..2f13c596 100644 --- a/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py +++ b/src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py @@ -11,7 +11,7 @@ from alembic import op import sqlalchemy as sa -from app.alembic.replaceable_objects.main import ReplaceableTrigger +from backend_api.app.alembic.replaceable_objects.main import ReplaceableTrigger # revision identifiers, used by Alembic. revision: str = "e38727349f3f" diff --git a/src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py b/src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py new file mode 100644 index 00000000..3d53b103 --- /dev/null +++ b/src/backend_api/app/alembic/versions/ee1239y6yfda_split_currency_table_into_prices_and_types.py @@ -0,0 +1,130 @@ +"""optimize item table + +Revision ID: ee1239y6yfda +Revises: 965e766db0a0 +Create Date: 2026-09-18 19:54:18.565656 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "ee1239y6yfda" +down_revision: Union[str, None] = "965e766db0a0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "currency_type", + sa.Column( + "currencyId", + sa.SmallInteger(), + sa.Identity(always=False, start=1, increment=1), + nullable=False, + primary_key=True, + ), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("tradeName", sa.Text(), nullable=False), + ) + + op.execute(""" + INSERT INTO currency_type (name, "tradeName") + SELECT DISTINCT 'TEMP_NAME', c."tradeName" + FROM currency c; + """) + + op.create_table( + "currency_price", + sa.Column( + "currencyId", + sa.SmallInteger(), + nullable=False, + ), + sa.Column("leagueId", sa.SmallInteger(), nullable=False), + sa.Column("createdHoursSinceLaunch", sa.SmallInteger(), nullable=False), + sa.Column("valueInChaos", sa.Float(4), nullable=False), + sa.PrimaryKeyConstraint("currencyId", "leagueId", "createdHoursSinceLaunch"), + sa.ForeignKeyConstraint( + ["currencyId"], + ["currency_type.currencyId"], + ondelete="RESTRICT", + onupdate="CASCADE", + ), + sa.ForeignKeyConstraint( + ["leagueId"], ["league.leagueId"], ondelete="CASCADE", onupdate="CASCADE" + ), + ) + + op.execute(""" + INSERT INTO currency_price + SELECT + ct."currencyId", + c."leagueId", + c."createdHoursSinceLaunch", + c."valueInChaos" + FROM currency c + JOIN currency_type ct + ON c."tradeName" = ct."tradeName"; + """) + + op.drop_constraint("item_currencyId_fkey", "item", type_="foreignkey") + op.drop_constraint("item_currencyId_fkey1", "item", type_="foreignkey") + op.execute(""" + UPDATE item i + SET "currencyId"=ct."currencyId" + FROM currency c + JOIN currency_type ct + ON ct."tradeName" = c."tradeName" + WHERE i."currencyId" = c."currencyId"; + """) + + op.drop_constraint( + "unidentified_item_currencyId_fkey", "unidentified_item", type_="foreignkey" + ) + op.execute(""" + UPDATE unidentified_item ui + SET "currencyId"=ct."currencyId" + FROM currency c + JOIN currency_type ct + ON ct."tradeName" = c."tradeName" + WHERE ui."currencyId" = c."currencyId"; + """) + + op.drop_table("currency") + + op.alter_column( + "item", + "currencyId", + existing_type=sa.Integer(), + type_=sa.SmallInteger(), + ) + op.create_foreign_key( + None, + "item", + "currency_type", + ["currencyId"], + ["currencyId"], + ) + + op.alter_column( + "unidentified_item", + "currencyId", + existing_type=sa.Integer(), + type_=sa.SmallInteger(), + ) + op.create_foreign_key( + None, + "unidentified_item", + "currency_type", + ["currencyId"], + ["currencyId"], + ) + + +def downgrade() -> None: + raise NotImplementedError("Downgrade is not supported for this migration") diff --git a/src/backend_api/app/api/api.py b/src/backend_api/app/api/api.py index 19175c6a..3d4f6b8b 100644 --- a/src/backend_api/app/api/api.py +++ b/src/backend_api/app/api/api.py @@ -1,13 +1,9 @@ -from fastapi import APIRouter - -from app.api.routes import ( +from backend_api.app.api.routes import ( currency, currency_prefix, item, item_base_type, item_base_type_prefix, - item_modifier, - item_modifier_prefix, item_prefix, league, league_prefix, @@ -24,6 +20,7 @@ unidentified_item, unidentified_item_prefix, ) +from fastapi import APIRouter api_router = APIRouter() @@ -36,11 +33,6 @@ prefix=f"/{item_base_type_prefix}", tags=[item_base_type_prefix], ) -api_router.include_router( - item_modifier.router, - prefix=f"/{item_modifier_prefix}", - tags=[item_modifier_prefix], -) api_router.include_router(item.router, prefix=f"/{item_prefix}", tags=[item_prefix]) api_router.include_router( league.router, prefix=f"/{league_prefix}", tags=[league_prefix] diff --git a/src/backend_api/app/api/api_message_util.py b/src/backend_api/app/api/api_message_util.py index d4fedd53..c546c595 100644 --- a/src/backend_api/app/api/api_message_util.py +++ b/src/backend_api/app/api/api_message_util.py @@ -1,9 +1,8 @@ -from pydantic import EmailStr - -from app.core.schemas.message import Message -from app.core.schemas.user import ( +from backend_api.app.core.schemas.message import Message +from backend_api.app.core.schemas.user import ( UsernameStr, ) +from pydantic import EmailStr def get_delete_return_msg( diff --git a/src/backend_api/app/api/deps.py b/src/backend_api/app/api/deps.py index e6b51a6a..4c42f0fb 100644 --- a/src/backend_api/app/api/deps.py +++ b/src/backend_api/app/api/deps.py @@ -1,23 +1,22 @@ from collections.abc import AsyncGenerator, Generator from typing import Annotated -from fastapi import Depends, Request -from fastapi.security import OAuth2PasswordBearer -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import Session -from starlette.datastructures import Headers - -from app.core.cache.user_cache import UserCache, UserCacheTokenType -from app.core.config import settings -from app.core.models.database import AsyncSessionLocal, SessionLocal -from app.core.models.models import User -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.exceptions import ( +from backend_api.app.core.cache.user_cache import UserCache, UserCacheTokenType +from backend_api.app.core.config import settings +from backend_api.app.core.models.database import AsyncSessionLocal, SessionLocal +from backend_api.app.core.models.models import User +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.exceptions import ( DbObjectDoesNotExistError, InvalidHeaderProvidedError, UserIsNotActiveError, UserWithNotEnoughPrivilegesError, ) +from fastapi import Depends, Request +from fastapi.security import OAuth2PasswordBearer +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Session +from starlette.datastructures import Headers reusable_oauth2 = OAuth2PasswordBearer( tokenUrl=f"{settings.API_V1_STR}/login/access-token", diff --git a/src/backend_api/app/api/routes/__init__.py b/src/backend_api/app/api/routes/__init__.py index 8cc7f8c0..8887a70a 100644 --- a/src/backend_api/app/api/routes/__init__.py +++ b/src/backend_api/app/api/routes/__init__.py @@ -1,12 +1,11 @@ -from app.api.routes.currency import currency_prefix -from app.api.routes.item_base_type import item_base_type_prefix -from app.api.routes.item_modifier import item_modifier_prefix -from app.api.routes.item import item_prefix -from app.api.routes.unidentified_item import unidentified_item_prefix -from app.api.routes.login import login_prefix -from app.api.routes.modifier import modifier_prefix -from app.api.routes.plot import plot_prefix -from app.api.routes.turnstile import turnstile_prefix -from app.api.routes.user import user_prefix -from app.api.routes.test import test_prefix -from app.api.routes.league import league_prefix +from backend_api.app.api.routes.currency import currency_prefix +from backend_api.app.api.routes.item import item_prefix +from backend_api.app.api.routes.item_base_type import item_base_type_prefix +from backend_api.app.api.routes.league import league_prefix +from backend_api.app.api.routes.login import login_prefix +from backend_api.app.api.routes.modifier import modifier_prefix +from backend_api.app.api.routes.plot import plot_prefix +from backend_api.app.api.routes.test import test_prefix +from backend_api.app.api.routes.turnstile import turnstile_prefix +from backend_api.app.api.routes.unidentified_item import unidentified_item_prefix +from backend_api.app.api.routes.user import user_prefix diff --git a/src/backend_api/app/api/routes/currency.py b/src/backend_api/app/api/routes/currency.py index d9e3ed03..62a5a74e 100644 --- a/src/backend_api/app/api/routes/currency.py +++ b/src/backend_api/app/api/routes/currency.py @@ -1,25 +1,21 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.api_message_util import ( - get_delete_return_msg, -) -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.models.models import Currency -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_user_rate_limits, ) -from app.core.schemas.currency import CurrencyQuery -from app.crud import CRUD_currency +from backend_api.app.core.schemas.currency import CurrencyQuery +from backend_api.app.crud import CRUD_currency +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() @@ -28,44 +24,35 @@ @router.get( - "/{currencyId}", - response_model=schemas.Currency, + "/price/", + response_model=schemas.CurrencyPrice | list[schemas.CurrencyPrice], dependencies=[ Depends(get_current_active_user), ], ) -@apply_user_rate_limits( - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_MINUTE, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_HOUR, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_DAY, -) -async def get_currency( - request: Request, # noqa: ARG001 - response: Response, # noqa: ARG001 - currencyId: int, +async def get_all_currency_prices( + filter_params: Annotated[FilterParams, Query()], db: Session = Depends(get_db), ): """ - Get currency by key and value for "currencyId". + Get all currencies. - Always returns one currency. + Returns a list of all currencies. """ - currency_map = {"currencyId": currencyId} - currency = await CRUD_currency.get(db=db, filter=currency_map) + all_currencies = await CRUD_currency.get_prices(db=db, filter_params=filter_params) - return currency + return all_currencies @router.get( - "/", - response_model=schemas.Currency | list[schemas.Currency], + "/type/", + response_model=schemas.CurrencyType | list[schemas.CurrencyType], dependencies=[ - Depends(get_current_active_superuser), + Depends(get_current_active_user), ], ) -async def get_all_currencies( +async def get_all_currency_types( filter_params: Annotated[FilterParams, Query()], db: Session = Depends(get_db), ): @@ -75,36 +62,33 @@ async def get_all_currencies( Returns a list of all currencies. """ - all_currencies = await CRUD_currency.get(db=db, filter_params=filter_params) + all_currencies = await CRUD_currency.get_types(db=db, filter_params=filter_params) return all_currencies @router.get( - "/latest_currency_id/", - response_model=int, - tags=["latest_currency_id"], + "/", + response_model=list[schemas.Currency], dependencies=[ Depends(get_current_active_user), ], ) -@apply_user_rate_limits( - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_MINUTE, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_HOUR, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_DAY, -) -async def get_latest_currency_id( - request: Request, # noqa: ARG001 - response: Response, # noqa: ARG001 +async def get_all_currency( + filter_params: Annotated[FilterParams, Query()], db: Session = Depends(get_db), ): """ - Get the latest currencyId, returns 1 if table is empty + Get all currencies. - Can only be used safely on an empty table or directly after an insertion. + Returns a list of all currencies. """ - return await CRUD_currency.get_latest_currency_id(db) + + all_currencies = await CRUD_currency.get_currency_from_query( + db=db, filter_params=filter_params + ) + + return all_currencies @router.get( @@ -136,9 +120,8 @@ async def get_latest_hours( @router.get( - "/latest_currencies/", + "/price/latest/", response_model=list[schemas.Currency], - tags=["latest_currencies"], dependencies=[ Depends(get_current_active_user), ], @@ -187,14 +170,14 @@ async def get_currency_from_query( @router.post( - "/", - response_model=schemas.CurrencyCreate | list[schemas.CurrencyCreate] | None, + "/type/", + response_model=schemas.CurrencyTypeCreate | list[schemas.CurrencyTypeCreate] | None, dependencies=[ Depends(get_current_active_superuser), ], ) -async def create_currency( - currency: schemas.CurrencyCreate | list[schemas.CurrencyCreate], +async def create_currency_type( + currency: schemas.CurrencyTypeCreate | list[schemas.CurrencyTypeCreate], return_nothing: bool | None = None, db: Session = Depends(get_db), ): @@ -204,59 +187,53 @@ async def create_currency( Returns the created currency or list of currencies. """ - return await CRUD_currency.create( + return await CRUD_currency.create_types( db=db, obj_in=currency, return_nothing=return_nothing ) -@router.put( - "/", - response_model=schemas.Currency, +@router.post( + "/price/", + response_model=schemas.CurrencyPriceCreate + | list[schemas.CurrencyPriceCreate] + | None, dependencies=[ Depends(get_current_active_superuser), ], ) -async def update_currency( - currencyId: int, - currency_update: schemas.CurrencyUpdate, +async def create_currency_price( + currency: schemas.CurrencyPriceCreate | list[schemas.CurrencyPriceCreate], + return_nothing: bool | None = None, db: Session = Depends(get_db), ): """ - Update a currency by key and value for "currencyId". + Create one or a list of currencies. - Returns the updated currency. + Returns the created currency or list of currencies. """ - currency_map = {"currencyId": currencyId} - currency = await CRUD_currency.get( - db=db, - filter=currency_map, + return await CRUD_currency.create_prices( + db=db, obj_in=currency, return_nothing=return_nothing ) - return await CRUD_currency.update(db_obj=currency, obj_in=currency_update, db=db) - -@router.delete( - "/", - response_model=str, - dependencies=[ - Depends(get_current_active_superuser), - ], +@router.put( + "/type/", + response_model=schemas.CurrencyType, + dependencies=[Depends(get_current_active_superuser)], ) -async def delete_currency( +async def update_modifier( currencyId: int, + currency_type_update: schemas.CurrencyTypeUpdate, db: Session = Depends(get_db), ): - """ - Delete a currency by key and value for "currencyId". - - Returns a message indicating the currency was deleted. - Always deletes one currency. - """ + currency_type_map = {"currencyId": currencyId} - currency_map = {"currencyId": currencyId} - await CRUD_currency.remove(db=db, filter=currency_map) + currency_type = await CRUD_currency.get_types( + db=db, + filter=currency_type_map, + ) - return get_delete_return_msg( - model_table_name=Currency.__tablename__, filter=currency_map - ).message + return await CRUD_currency.update_type( + db_obj=currency_type, obj_in=currency_type_update, db=db + ) diff --git a/src/backend_api/app/api/routes/item.py b/src/backend_api/app/api/routes/item.py index cef6074a..87d47725 100644 --- a/src/backend_api/app/api/routes/item.py +++ b/src/backend_api/app/api/routes/item.py @@ -1,19 +1,14 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy import text -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( +import backend_api.app.core.schemas as schemas +from backend_api.app.api.deps import ( get_current_active_superuser, - get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_user_rate_limits -from app.crud import CRUD_item +from backend_api.app.api.params import FilterParams +from backend_api.app.crud import CRUD_item +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session router = APIRouter() @@ -21,36 +16,6 @@ item_prefix = "item" -@router.get( - "/latest_item_id/", - response_model=int | None, - tags=["latest_item_id"], - dependencies=[Depends(get_current_active_user)], -) -@apply_user_rate_limits( - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_MINUTE, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_HOUR, - rate_limit_settings.DEFAULT_USER_RATE_LIMIT_DAY, -) -async def get_latest_item_id( - request: Request, # noqa: ARG001 - response: Response, # noqa: ARG001 - db: Session = Depends(get_db), -): - """ - Get the latest "itemId" - - Can only be used safely on an empty table or directly after an insertion. - """ - - result = db.execute(text("""SELECT MAX("itemId") FROM item""")).fetchone() - if not result or not result[0]: - return None - - return int(result[0]) - - @router.get( "/", response_model=schemas.Item | list[schemas.Item], @@ -73,12 +38,11 @@ async def get_all_items( @router.post( "/", - response_model=schemas.ItemCreate | list[schemas.ItemCreate] | None, + response_model=list[schemas.Item], dependencies=[Depends(get_current_active_superuser)], ) async def create_item( - item: schemas.ItemCreate | list[schemas.ItemCreate], - return_nothing: bool | None = None, + items: list[schemas.ItemCreate], db: Session = Depends(get_db), ): """ @@ -87,4 +51,32 @@ async def create_item( Returns the created item or list of items. """ - return await CRUD_item.create(db=db, obj_in=item, return_nothing=return_nothing) + return await CRUD_item.create_items(db=db, new_items=items) + + +@router.patch( + "/availability/", + response_model=list[schemas.ItemAvailability], + dependencies=[Depends(get_current_active_superuser)], +) +async def patch_expired_availability( + expired_availability: list[schemas.ItemAvailabilityExpired], + db: Session = Depends(get_db), +): + return await CRUD_item.patch_expired_availability( + db, expired_availability=expired_availability + ) + + +@router.put( + "/availability/", + response_model=list[schemas.ItemAvailability], + dependencies=[Depends(get_current_active_superuser)], +) +async def update_availability( + updated_availability: list[schemas.ItemAvailabilityUpdated], + db: Session = Depends(get_db), +): + return await CRUD_item.update_availability( + db, updated_availability=updated_availability + ) diff --git a/src/backend_api/app/api/routes/item_base_type.py b/src/backend_api/app/api/routes/item_base_type.py index 1b7a6459..ba671ca5 100644 --- a/src/backend_api/app/api/routes/item_base_type.py +++ b/src/backend_api/app/api/routes/item_base_type.py @@ -1,25 +1,25 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_delete_return_msg, ) -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.models.models import ItemBaseType -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.models.models import ItemBaseType +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.crud import CRUD_itemBaseType +from backend_api.app.crud import CRUD_itemBaseType +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() diff --git a/src/backend_api/app/api/routes/item_modifier.py b/src/backend_api/app/api/routes/item_modifier.py deleted file mode 100644 index 92154a3c..00000000 --- a/src/backend_api/app/api/routes/item_modifier.py +++ /dev/null @@ -1,60 +0,0 @@ -from typing import Annotated - -from fastapi import APIRouter, Depends, Query -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( - get_current_active_superuser, - get_db, -) -from app.api.params import FilterParams -from app.crud import CRUD_itemModifier - -router = APIRouter() - - -item_modifier_prefix = "itemModifier" - - -@router.get( - "/", - response_model=schemas.ItemModifier | list[schemas.ItemModifier], - dependencies=[Depends(get_current_active_superuser)], -) -async def get_all_item_modifiers( - filter_params: Annotated[FilterParams, Query()], - db: Session = Depends(get_db), -): - """ - Get all item modifiers. - - Returns a list of all item modifiers. - """ - - all_itemModifiers = await CRUD_itemModifier.get(db=db, filter_params=filter_params) - - return all_itemModifiers - - -@router.post( - "/", - response_model=schemas.ItemModifierCreate | list[schemas.ItemModifierCreate] | None, - dependencies=[Depends(get_current_active_superuser)], -) -async def create_item_modifier( - itemModifier: schemas.ItemModifierCreate | list[schemas.ItemModifierCreate], - return_nothing: bool | None = None, - db: Session = Depends(get_db), -): - """ - Create one or a list item modifiers. - - Returns the created item modifier or list of item modifiers. - """ - - return await CRUD_itemModifier.create( - db=db, - obj_in=itemModifier, - return_nothing=return_nothing, - ) diff --git a/src/backend_api/app/api/routes/league.py b/src/backend_api/app/api/routes/league.py index af8877bf..2f7b234a 100644 --- a/src/backend_api/app/api/routes/league.py +++ b/src/backend_api/app/api/routes/league.py @@ -1,21 +1,21 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.crud import CRUD_league +from backend_api.app.crud import CRUD_league +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() diff --git a/src/backend_api/app/api/routes/login.py b/src/backend_api/app/api/routes/login.py index 9a666e69..f961dc4a 100644 --- a/src/backend_api/app/api/routes/login.py +++ b/src/backend_api/app/api/routes/login.py @@ -1,24 +1,23 @@ # From FastAPI Fullstack Template https://github.com/fastapi/full-stack-fastapi-template/blob/master/backend/app/api/routes/login.py from typing import Annotated -from fastapi import APIRouter, Depends, Request, Response -from fastapi.security import OAuth2PasswordRequestForm -from sqlalchemy.orm import Session - -from app.api.deps import ( +from backend_api.app.api.deps import ( UserCacheSession, get_db, ) -from app.core.config import settings -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.core.config import settings +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, ) -from app.core.schemas import Token -from app.crud import CRUD_user -from app.exceptions import ( +from backend_api.app.core.schemas import Token +from backend_api.app.crud import CRUD_user +from backend_api.app.exceptions import ( BadLoginCredentialsError, ) +from fastapi import APIRouter, Depends, Request, Response +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy.orm import Session router = APIRouter() diff --git a/src/backend_api/app/api/routes/modifier.py b/src/backend_api/app/api/routes/modifier.py index cd83a6e2..81ea9471 100644 --- a/src/backend_api/app/api/routes/modifier.py +++ b/src/backend_api/app/api/routes/modifier.py @@ -1,25 +1,25 @@ from typing import Annotated -from fastapi import APIRouter, Depends, Query, Request, Response -from sqlalchemy.orm import Session - -import app.core.schemas as schemas -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_delete_return_msg, ) -from app.api.deps import ( +from backend_api.app.api.deps import ( get_current_active_superuser, get_current_active_user, get_db, ) -from app.api.params import FilterParams -from app.core.models.models import Modifier -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.api.params import FilterParams +from backend_api.app.core.models.models import Modifier +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.crud import CRUD_modifier +from backend_api.app.crud import CRUD_modifier +from fastapi import APIRouter, Depends, Query, Request, Response +from sqlalchemy.orm import Session + +import app.core.schemas as schemas router = APIRouter() @@ -88,9 +88,8 @@ async def get_all_modifiers( @router.get( - "/grouped_modifiers_by_effect/", - response_model=schemas.GroupedModifierByEffect - | list[schemas.GroupedModifierByEffect], + "/grouped/", + response_model=list[schemas.GroupedModifier], ) @apply_ip_rate_limits( rate_limit_settings.DEFAULT_USER_RATE_LIMIT_SECOND, @@ -109,43 +108,33 @@ async def get_grouped_modifier_by_effect( Returns a list of all grouped modifiers by effect. """ - all_grouped_modifiers_by_effect = ( - await CRUD_modifier.get_grouped_modifier_by_effect(db=db) - ) + all_grouped_modifiers_by_effect = await CRUD_modifier.get_grouped_modifiers(db=db) return all_grouped_modifiers_by_effect @router.post( "/", - response_model=schemas.ModifierCreate | list[schemas.ModifierCreate] | None, + response_model=list[schemas.Modifier], dependencies=[Depends(get_current_active_superuser)], ) async def create_modifier( - modifier: schemas.ModifierCreate | list[schemas.ModifierCreate], - return_nothing: bool | None = None, + modifiers: list[schemas.ModifierCreate], db: Session = Depends(get_db), ): """ - Create one or a list of new modifiers. - - Returns the created modifier or list of modifiers. + Create a list of new modifiers. """ - return await CRUD_modifier.create( - db=db, obj_in=modifier, return_nothing=return_nothing - ) + return await CRUD_modifier.create(db=db, modifiers=modifiers) @router.put( "/", - response_model=schemas.Modifier, dependencies=[Depends(get_current_active_superuser)], ) async def update_modifier( - modifierId: int, - position: int, - modifier_update: schemas.ModifierUpdate, + modifier: schemas.ModifierUpdate, db: Session = Depends(get_db), ): """ @@ -153,15 +142,7 @@ async def update_modifier( Returns the updated modifier. """ - - modifier_map = {"modifierId": modifierId, "position": position} - - modifier = await CRUD_modifier.get( - db=db, - filter=modifier_map, - ) - - return await CRUD_modifier.update(db_obj=modifier, obj_in=modifier_update, db=db) + return await CRUD_modifier.update(db=db, modifier=modifier) @router.delete( diff --git a/src/backend_api/app/api/routes/plot.py b/src/backend_api/app/api/routes/plot.py index 4ce04d08..d5972f9d 100644 --- a/src/backend_api/app/api/routes/plot.py +++ b/src/backend_api/app/api/routes/plot.py @@ -1,15 +1,14 @@ -from fastapi import APIRouter, Depends, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import ( +from backend_api.app.api.deps import ( get_async_db, get_user_ip_from_header, ) -from app.core.rate_limit.custom_rate_limiter import RateSpec -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_custom_rate_limit -from app.core.schemas.plot import PlotData, PlotQuery -from app.plotting import configure_plotter_by_query, plotter_service +from backend_api.app.core.rate_limit.custom_rate_limiter import RateSpec +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_custom_rate_limit +from backend_api.app.core.schemas.plot import PlotData, PlotQuery +from backend_api.app.plotting import configure_plotter_by_query, plotter_service +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter() diff --git a/src/backend_api/app/api/routes/test.py b/src/backend_api/app/api/routes/test.py index 85548b95..568a36a1 100644 --- a/src/backend_api/app/api/routes/test.py +++ b/src/backend_api/app/api/routes/test.py @@ -1,22 +1,23 @@ from uuid import uuid4 +from backend_api.app.api.deps import ( + get_current_active_superuser, + get_db, +) +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Item as model_Item +from backend_api.app.core.models.models import User as model_User +from backend_api.app.core.schemas import ItemCreate, User +from backend_api.app.crud import CRUD_user +from backend_api.app.exceptions.model_exceptions.test_exception import ( + OnlyAvailableInLocalEnvError, +) +from backend_api.app.logs.logger import test_logger as logger from fastapi import APIRouter, Depends, HTTPException from httpx import ASGITransport, AsyncClient from sqlalchemy import insert from sqlalchemy.orm import Session -from app.api.deps import ( - get_current_active_superuser, - get_db, -) -from app.core.config import settings -from app.core.models.models import Item as model_Item -from app.core.models.models import User as model_User -from app.core.schemas import ItemCreate, User -from app.crud import CRUD_user -from app.exceptions.model_exceptions.test_exception import OnlyAvailableInLocalEnvError -from app.logs.logger import test_logger as logger - router = APIRouter() @@ -133,7 +134,7 @@ async def bulk_insert_users_and_verify(count: int, db: Session = Depends(get_db) logger.debug("Verifying users") # Verify users by making an internal request to the login route - from app.main import app + from backend_api.app.main import app tokens = [] async with AsyncClient( diff --git a/src/backend_api/app/api/routes/turnstile.py b/src/backend_api/app/api/routes/turnstile.py index 9d160f91..1f448905 100644 --- a/src/backend_api/app/api/routes/turnstile.py +++ b/src/backend_api/app/api/routes/turnstile.py @@ -1,13 +1,13 @@ +from backend_api.app.api.deps import ( + get_user_ip_from_header, +) +from backend_api.app.core.rate_limit.custom_rate_limiter import RateSpec +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_custom_rate_limit +from backend_api.app.validation import turnstile_validation_tool from fastapi import APIRouter, Request import app.core.schemas as schemas -from app.api.deps import ( - get_user_ip_from_header, -) -from app.core.rate_limit.custom_rate_limiter import RateSpec -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_custom_rate_limit -from app.validation import turnstile_validation_tool router = APIRouter() diff --git a/src/backend_api/app/api/routes/unidentified_item.py b/src/backend_api/app/api/routes/unidentified_item.py index cc55c311..33a9d057 100644 --- a/src/backend_api/app/api/routes/unidentified_item.py +++ b/src/backend_api/app/api/routes/unidentified_item.py @@ -1,19 +1,19 @@ from typing import Annotated +from backend_api.app.api.deps import ( + get_current_active_superuser, + get_current_active_user, + get_db, +) +from backend_api.app.api.params import FilterParams +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import apply_user_rate_limits +from backend_api.app.crud import CRUD_unidentifiedItem from fastapi import APIRouter, Depends, Query, Request, Response from sqlalchemy import text from sqlalchemy.orm import Session import app.core.schemas as schemas -from app.api.deps import ( - get_current_active_superuser, - get_current_active_user, - get_db, -) -from app.api.params import FilterParams -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import apply_user_rate_limits -from app.crud import CRUD_unidentifiedItem router = APIRouter() diff --git a/src/backend_api/app/api/routes/user.py b/src/backend_api/app/api/routes/user.py index a0e099f3..224df60e 100644 --- a/src/backend_api/app/api/routes/user.py +++ b/src/backend_api/app/api/routes/user.py @@ -1,10 +1,7 @@ import uuid from typing import Any -from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response -from sqlalchemy.orm import Session - -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_activation_token_confirmation_sent_msg, get_delete_return_msg, get_set_rate_limit_tier_success_msg, @@ -15,7 +12,7 @@ get_user_update_me_confirmation_sent_msg, get_user_update_me_success_msg, ) -from app.api.deps import ( +from backend_api.app.api.deps import ( CurrentUser, CurrentUserNotActive, UserCacheRegisterSession, @@ -24,14 +21,14 @@ get_current_active_user, get_db, ) -from app.core.config import settings -from app.core.models.models import User -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.core.rate_limit.rate_limiters import ( +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import User +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.core.rate_limit.rate_limiters import ( apply_ip_rate_limits, apply_user_rate_limits, ) -from app.core.schemas import ( +from backend_api.app.core.schemas import ( Message, UpdatePassword, UserCreate, @@ -40,10 +37,10 @@ UsersPublic, UserUpdate, ) -from app.core.schemas.token import Token -from app.core.schemas.user import UserUpdateMe -from app.crud import CRUD_user -from app.exceptions import ( +from backend_api.app.core.schemas.token import Token +from backend_api.app.core.schemas.user import UserUpdateMe +from backend_api.app.crud import CRUD_user +from backend_api.app.exceptions import ( DbObjectAlreadyExistsError, DbObjectDoesNotExistError, InvalidTokenError, @@ -51,13 +48,13 @@ SuperUserNotAllowedToDeleteSelfError, UserWithNotEnoughPrivilegesError, ) -from app.exceptions.model_exceptions.user_login_exception import ( +from backend_api.app.exceptions.model_exceptions.user_login_exception import ( UpdateExisitingMeValuesError, UserEmailRequiredError, UserIsAlreadyActiveError, UserUsernameRequiredError, ) -from app.utils.user import ( +from backend_api.app.utils.user import ( generate_email_changed_notify_email, generate_new_account_email, generate_password_changed_notify_email, @@ -65,6 +62,8 @@ generate_user_registration_email, send_email, ) +from fastapi import APIRouter, BackgroundTasks, Depends, Request, Response +from sqlalchemy.orm import Session router = APIRouter() diff --git a/src/backend_api/app/backend_pre_start.py b/src/backend_api/app/backend_pre_start.py index fbc48689..fca12571 100644 --- a/src/backend_api/app/backend_pre_start.py +++ b/src/backend_api/app/backend_pre_start.py @@ -1,11 +1,10 @@ import logging +from backend_api.app.core.models.database import SessionLocal +from backend_api.app.logs.logger import logger from sqlalchemy import select from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed -from app.core.models.database import SessionLocal -from app.logs.logger import logger - max_tries = 60 * 5 # 5 minutes wait_seconds = 1 diff --git a/src/backend_api/app/core/cache/cache.py b/src/backend_api/app/core/cache/cache.py index 6c6e96c4..9945306d 100644 --- a/src/backend_api/app/core/cache/cache.py +++ b/src/backend_api/app/core/cache/cache.py @@ -1,5 +1,4 @@ import redis.asyncio as Redis +from backend_api.app.core.config import settings -from app.core.config import settings - -cache = Redis.from_url(str(settings.CACHE_URI), decode_responses=True) +cache = Redis.from_url(str(settings.API_CACHE_URI), decode_responses=True) diff --git a/src/backend_api/app/core/cache/user_cache.py b/src/backend_api/app/core/cache/user_cache.py index a35e9742..960e021c 100644 --- a/src/backend_api/app/core/cache/user_cache.py +++ b/src/backend_api/app/core/cache/user_cache.py @@ -2,15 +2,14 @@ from typing import Any from uuid import UUID, uuid4 -from pydantic import TypeAdapter - -from app.core.cache.cache import cache -from app.core.models.models import User as model_User -from app.core.schemas.user import UserInCache -from app.exceptions import ( +from backend_api.app.core.cache.cache import cache +from backend_api.app.core.models.models import User as model_User +from backend_api.app.core.schemas.user import UserInCache +from backend_api.app.exceptions import ( InvalidCacheUpdateParamsError, InvalidTokenError, ) +from pydantic import TypeAdapter class UserCacheTokenType(StrEnum): diff --git a/src/backend_api/app/core/config.py b/src/backend_api/app/core/config.py index 971b2cf0..be359fa2 100644 --- a/src/backend_api/app/core/config.py +++ b/src/backend_api/app/core/config.py @@ -83,18 +83,18 @@ def ASYNC_DATABASE_URI(self) -> PostgresDsn: REDIS_PORT: int = 6379 REDIS_SERVER: str - REDIS_CACHE: str = str(0) + REDIS_BACKEND_API: int = 0 REDIS_PASSWORD: str = "" @computed_field # type: ignore[prop-decorator] @property - def CACHE_URI(self) -> AnyUrl: + def API_CACHE_URI(self) -> AnyUrl: return MultiHostUrl.build( scheme="redis", password=self.REDIS_PASSWORD, host=self.REDIS_SERVER, port=self.REDIS_PORT, - path=self.REDIS_CACHE, + path=f"{self.REDIS_BACKEND_API}", ) SMTP_TLS: bool = True diff --git a/src/backend_api/app/core/models/database.py b/src/backend_api/app/core/models/database.py index 763fb17a..811759c9 100644 --- a/src/backend_api/app/core/models/database.py +++ b/src/backend_api/app/core/models/database.py @@ -1,10 +1,9 @@ +from backend_api.app.core.config import settings from sqlalchemy import create_engine from sqlalchemy.ext.asyncio import AsyncAttrs, async_sessionmaker, create_async_engine from sqlalchemy.inspection import inspect from sqlalchemy.orm import DeclarativeBase, sessionmaker -from app.core.config import settings - engine = create_engine(str(settings.DATABASE_URI)) async_engine = create_async_engine( # This engine is used for plotting queries str(settings.ASYNC_DATABASE_URI) diff --git a/src/backend_api/app/core/models/init_db.py b/src/backend_api/app/core/models/init_db.py index d2e37d9d..ec61aeb8 100644 --- a/src/backend_api/app/core/models/init_db.py +++ b/src/backend_api/app/core/models/init_db.py @@ -1,11 +1,10 @@ +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import User +from backend_api.app.core.schemas.user import UserCreate +from backend_api.app.crud import CRUD_user from sqlalchemy import select from sqlalchemy.orm import Session -from app.core.config import settings -from app.core.models.models import User -from app.core.schemas.user import UserCreate -from app.crud import CRUD_user - def init_db(session: Session) -> None: user = session.execute( diff --git a/src/backend_api/app/core/models/models.py b/src/backend_api/app/core/models/models.py index bbe6f325..f2001f61 100644 --- a/src/backend_api/app/core/models/models.py +++ b/src/backend_api/app/core/models/models.py @@ -1,8 +1,9 @@ import uuid from datetime import datetime +from backend_api.app.core.models.database import Base from sqlalchemy import ( - BigInteger, + ARRAY, Boolean, CheckConstraint, DateTime, @@ -16,13 +17,12 @@ SmallInteger, String, Text, + UniqueConstraint, func, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column -from app.core.models.database import Base - class League(Base): __tablename__ = "league" @@ -36,18 +36,32 @@ class League(Base): version: Mapped[float] = mapped_column(Float, nullable=False) -class Currency(Base): - __tablename__ = "currency" - - currencyId: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True) - createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) - valueInChaos: Mapped[float] = mapped_column(Float(4), nullable=False) +class CurrencyType(Base): + __tablename__ = "currency_type" + currencyId: Mapped[int] = mapped_column(SmallInteger, Identity(), primary_key=True) + name: Mapped[str] = mapped_column(Text, nullable=False) tradeName: Mapped[str] = mapped_column(Text, nullable=False) - leagueId: Mapped[SmallInteger] = mapped_column( + + +class CurrencyPrice(Base): + __tablename__ = "currency_price" + + currencyId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey("currency_type.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), + nullable=False, + ) + leagueId: Mapped[int] = mapped_column( SmallInteger, ForeignKey("league.leagueId", ondelete="RESTRICT", onupdate="CASCADE"), nullable=False, ) + createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) + valueInChaos: Mapped[float] = mapped_column(Float(4), nullable=False) + + __table_args__ = ( + PrimaryKeyConstraint("currencyId", "leagueId", "createdHoursSinceLaunch"), + ) class ItemBaseType(Base): @@ -68,49 +82,71 @@ class ItemBaseType(Base): relatedUniques: Mapped[str | None] = mapped_column(Text) -class _ItemBase: - name: Mapped[str | None] = mapped_column(Text, nullable=False) - itemBaseTypeId: Mapped[int] = mapped_column( - SmallInteger, - ForeignKey( - "item_base_type.itemBaseTypeId", ondelete="RESTRICT", onupdate="CASCADE" - ), - nullable=False, +class ItemAvailability(Base): + __tablename__ = "item_availability" + + availabilityId: Mapped[int] = mapped_column( + Integer, primary_key=True, autoincrement=True ) - createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) - leagueId: Mapped[SmallInteger] = mapped_column( - SmallInteger, - ForeignKey("league.leagueId", ondelete="RESTRICT", onupdate="CASCADE"), + itemId: Mapped[str] = mapped_column( + Integer, + ForeignKey("item.itemId", ondelete="CASCADE", onupdate="CASCADE"), nullable=False, ) - itemId: Mapped[int] = mapped_column( - BigInteger, - Identity(start=1, increment=1, always=True), - primary_key=True, # Primary key constraint gets removed on hypertable creation - ) + currencyId: Mapped[int] = mapped_column( Integer, - ForeignKey("currency.currencyId", ondelete="RESTRICT"), - index=True, + ForeignKey("currency_type.currencyId", ondelete="RESTRICT", onupdate="CASCADE"), nullable=False, ) - ilvl: Mapped[int] = mapped_column(SmallInteger, nullable=False) currencyAmount: Mapped[float] = mapped_column(Float(4), nullable=False) - rarity: Mapped[str] = mapped_column(Text, nullable=False) + validFrom: Mapped[int] = mapped_column(SmallInteger, nullable=False) + validTo: Mapped[int | None] = mapped_column(SmallInteger, nullable=True) -class Item(_ItemBase, Base): - # Hypertable - # For hypertable specs, see alembic revision `cc29b89156db' + isAsync: Mapped[bool | None] = mapped_column(Boolean, nullable=False) + + __table_args__ = ( + Index("ix_item_id_valid_from", "itemId", "validFrom"), + UniqueConstraint("itemId", "validFrom"), + CheckConstraint( + """ + item_availability."validTo" IS NULL + OR item_availability."validTo" > item_availability."validFrom" + """, + name="check_positive_duration", + ), + ) + + +class Item(Base): __tablename__ = "item" - # TODO do something about None and make it not nullable - gameItemId: Mapped[str | None] = mapped_column(Text) - prefixes: Mapped[int | None] = mapped_column(SmallInteger) - suffixes: Mapped[int | None] = mapped_column(SmallInteger) - foilVariation: Mapped[int | None] = mapped_column(SmallInteger) + itemId: Mapped[int] = mapped_column( + Integer, + Identity(start=1, increment=1), + primary_key=True, + ) + gameItemId: Mapped[str | None] = mapped_column(Text, nullable=False) + leagueId: Mapped[SmallInteger] = mapped_column( + SmallInteger, + ForeignKey("league.leagueId", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + ) + firstObserved: Mapped[int] = mapped_column(SmallInteger, nullable=False) + name: Mapped[str] = mapped_column(Text, nullable=False) + itemBaseTypeId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey( + "item_base_type.itemBaseTypeId", ondelete="RESTRICT", onupdate="CASCADE" + ), + nullable=False, + ) + ilvl: Mapped[int] = mapped_column(SmallInteger, nullable=False) + rarity: Mapped[str] = mapped_column(Text, nullable=False) + identified: Mapped[bool] = mapped_column(Boolean, nullable=False) corrupted: Mapped[bool | None] = mapped_column(Boolean) - delve: Mapped[bool | None] = mapped_column(Boolean) + fractured: Mapped[bool | None] = mapped_column(Boolean) synthesised: Mapped[bool | None] = mapped_column(Boolean) replica: Mapped[bool | None] = mapped_column(Boolean) @@ -122,15 +158,48 @@ class Item(_ItemBase, Base): __table_args__ = ( Index( - "ix_item_name_itemBaseTypeId_createdHoursSinceLaunch_leagueId", - "name", - "itemBaseTypeId", - "createdHoursSinceLaunch", + "ix_item_leagueId_itemBaseTypeId", "leagueId", + "itemBaseTypeId", + ), + UniqueConstraint( + "gameItemId", "leagueId", name="uq_item_game_item_id_league_id" ), ) +class _ItemBase: + name: Mapped[str | None] = mapped_column(Text, nullable=False) + itemBaseTypeId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey( + "item_base_type.itemBaseTypeId", ondelete="RESTRICT", onupdate="CASCADE" + ), + nullable=False, + ) + createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) + leagueId: Mapped[SmallInteger] = mapped_column( + SmallInteger, + ForeignKey("league.leagueId", ondelete="RESTRICT", onupdate="CASCADE"), + nullable=False, + ) + itemId: Mapped[int] = mapped_column( + Integer, + Identity(start=1, increment=1, always=True), + primary_key=True, # Primary key constraint gets removed on hypertable creation + ) + + currencyId: Mapped[int] = mapped_column( + Integer, + ForeignKey("currency_type.currencyId", ondelete="RESTRICT"), + index=True, + nullable=False, + ) + ilvl: Mapped[int] = mapped_column(SmallInteger, nullable=False) + currencyAmount: Mapped[float] = mapped_column(Float(4), nullable=False) + rarity: Mapped[str] = mapped_column(Text, nullable=False) + + class UnidentifiedItem(_ItemBase, Base): """ IS-A item relation, couldn't be bothered finding the actual way to implement it. @@ -163,10 +232,14 @@ class UnidentifiedItem(_ItemBase, Base): class Modifier(Base): __tablename__ = "modifier" - modifierId: Mapped[int] = mapped_column(SmallInteger, nullable=False) - position: Mapped[int] = mapped_column(SmallInteger, nullable=False) - minRoll: Mapped[float | None] = mapped_column(Float(4)) - maxRoll: Mapped[float | None] = mapped_column(Float(4)) + modifierId: Mapped[int] = mapped_column(SmallInteger, primary_key=True) + + static: Mapped[bool | None] = mapped_column(Boolean) + effect: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + regex: Mapped[str | None] = mapped_column(Text) + + relatedUniques: Mapped[str | None] = mapped_column(Text) + implicit: Mapped[bool | None] = mapped_column(Boolean) explicit: Mapped[bool | None] = mapped_column(Boolean) delve: Mapped[bool | None] = mapped_column(Boolean) @@ -176,11 +249,7 @@ class Modifier(Base): corrupted: Mapped[bool | None] = mapped_column(Boolean) enchanted: Mapped[bool | None] = mapped_column(Boolean) veiled: Mapped[bool | None] = mapped_column(Boolean) - static: Mapped[bool | None] = mapped_column(Boolean) - effect: Mapped[str] = mapped_column(Text, nullable=False) - relatedUniques: Mapped[str | None] = mapped_column(Text) - textRolls: Mapped[str | None] = mapped_column(Text) - regex: Mapped[str | None] = mapped_column(Text) + createdAt: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=func.now(), nullable=False ) @@ -190,29 +259,15 @@ class Modifier(Base): ) __table_args__ = ( - PrimaryKeyConstraint("modifierId", "position"), CheckConstraint( """ CASE WHEN (modifier.static = TRUE) THEN ( - (modifier."minRoll" IS NULL AND modifier."maxRoll" IS NULL) - AND modifier."textRolls" IS NULL - AND modifier.regex IS NULL + modifier.regex IS NULL ) ELSE ( - ( - ( - (modifier."minRoll" IS NOT NULL AND modifier."maxRoll" IS NOT NULL) - AND modifier."textRolls" IS NULL - ) - OR - ( - (modifier."minRoll" IS NULL AND modifier."maxRoll" IS NULL) - AND modifier."textRolls" IS NOT NULL - ) - ) - AND modifier.regex IS NOT NULL + modifier.regex IS NOT NULL ) END """, @@ -232,18 +287,38 @@ class Modifier(Base): """, name="check_modifier_if_not_static_then_modifier_contains_hashtag", ), + ) + + +class ModifierRoll(Base): + __tablename__ = "modifier_roll" + + modifierId: Mapped[int] = mapped_column( + SmallInteger, + ForeignKey("modifier.modifierId", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + index=True, + ) + position: Mapped[int] = mapped_column(SmallInteger, nullable=False) + minRoll: Mapped[float | None] = mapped_column(Float(4)) + maxRoll: Mapped[float | None] = mapped_column(Float(4)) + textRolls: Mapped[list[str] | None] = mapped_column(ARRAY(Text())) + + __table_args__ = ( + PrimaryKeyConstraint("modifierId", "position"), CheckConstraint( - """ modifier."maxRoll" >= modifier."minRoll" """, + """ modifier_roll."maxRoll" >= modifier_roll."minRoll" """, name="check_modifier_maxRoll_greaterThan_minRoll", ), ) class ItemModifier(Base): - # Hypertable - # For hypertable specs, see alembic revision `cc29b89156db' - __tablename__ = "item_modifier" + itemId: Mapped[int] = mapped_column( + Integer, + nullable=False, + ) modifierId: Mapped[int] = mapped_column( SmallInteger, @@ -253,27 +328,24 @@ class ItemModifier(Base): SmallInteger, nullable=False, ) - createdHoursSinceLaunch: Mapped[int] = mapped_column(SmallInteger, nullable=False) - itemId: Mapped[int] = mapped_column( - BigInteger, + instance: Mapped[int] = mapped_column( + SmallInteger, nullable=False, - primary_key=True, # Primary key constraint gets removed on hypertable creation - ) + ) # a modifier can appear multiple times on an item (eg. forbidden shako) roll: Mapped[float | None] = mapped_column( Float(4), ) __table_args__ = ( + PrimaryKeyConstraint("itemId", "modifierId", "position"), ForeignKeyConstraint( ["modifierId", "position"], - ["modifier.modifierId", "modifier.position"], + ["modifier_roll.modifierId", "modifier_roll.position"], ondelete="CASCADE", onupdate="CASCADE", ), Index( - "ix_item_modifierId_createdHoursSinceLaunch_roll_itemId", + "ix_item_modifierId_itemId", "modifierId", - "createdHoursSinceLaunch", - "roll", "itemId", ), ) diff --git a/src/backend_api/app/core/rate_limit/custom_rate_limiter.py b/src/backend_api/app/core/rate_limit/custom_rate_limiter.py index 9aaa2a38..77f73fed 100644 --- a/src/backend_api/app/core/rate_limit/custom_rate_limiter.py +++ b/src/backend_api/app/core/rate_limit/custom_rate_limiter.py @@ -9,10 +9,11 @@ AnyRedis, pipeline_expire, ) +from backend_api.app.exceptions.model_exceptions.rate_limit_exception import ( + RateLimitExceededError, +) from typing_extensions import final -from app.exceptions.model_exceptions.rate_limit_exception import RateLimitExceededError - #: These aliases makes our code more readable. _Seconds: TypeAlias = int diff --git a/src/backend_api/app/core/rate_limit/rate_limiters.py b/src/backend_api/app/core/rate_limit/rate_limiters.py index 1325e7bd..6f3f34ce 100644 --- a/src/backend_api/app/core/rate_limit/rate_limiters.py +++ b/src/backend_api/app/core/rate_limit/rate_limiters.py @@ -2,14 +2,13 @@ from contextlib import asynccontextmanager from typing import Any +from backend_api.app.api.deps import get_user_ip_from_header, get_username_by_request +from backend_api.app.core.cache.cache import cache +from backend_api.app.core.config import settings +from backend_api.app.core.rate_limit.custom_rate_limiter import RateLimiter, RateSpec +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings from slowapi import Limiter -from app.api.deps import get_username_by_request, get_user_ip_from_header -from app.core.cache.cache import cache -from app.core.config import settings -from app.core.rate_limit.custom_rate_limiter import RateLimiter, RateSpec -from app.core.rate_limit.rate_limit_config import rate_limit_settings - def default_limit_provider() -> list[str]: return [ @@ -24,7 +23,7 @@ def default_limit_provider() -> list[str]: limiter_user = Limiter( key_func=get_username_by_request, default_limits=default_limit_provider(), - storage_uri=str(settings.CACHE_URI), + storage_uri=str(settings.API_CACHE_URI), headers_enabled=True, enabled=settings.RATE_LIMIT, ) @@ -33,7 +32,7 @@ def default_limit_provider() -> list[str]: limiter_ip = Limiter( key_func=get_user_ip_from_header, default_limits=default_limit_provider(), - storage_uri=str(settings.CACHE_URI), + storage_uri=str(settings.API_CACHE_URI), headers_enabled=True, enabled=settings.RATE_LIMIT, ) diff --git a/src/backend_api/app/core/schemas/__init__.py b/src/backend_api/app/core/schemas/__init__.py index 4d69acff..7af82163 100644 --- a/src/backend_api/app/core/schemas/__init__.py +++ b/src/backend_api/app/core/schemas/__init__.py @@ -1,7 +1,25 @@ "All schemas are imported here and then exported to the main file" -from .currency import Currency, CurrencyCreate, CurrencyInDB, CurrencyUpdate -from .item import Item, ItemCreate, ItemInDB, ItemUpdate +from .currency import ( + Currency, + CurrencyPrice, + CurrencyPriceCreate, + CurrencyPriceInDB, + CurrencyPriceUpdate, + CurrencyType, + CurrencyTypeCreate, + CurrencyTypeInDB, + CurrencyTypeUpdate, +) +from .item import ( + Item, + ItemAvailability, + ItemAvailabilityExpired, + ItemAvailabilityUpdated, + ItemCreate, + ItemQuery, + ItemUpdate, +) from .item_base_type import ( ItemBaseType, ItemBaseTypeCreate, @@ -11,8 +29,6 @@ from .item_modifier import ( ItemModifier, ItemModifierCreate, - ItemModifierInDB, - ItemModifierUpdate, ) from .league import ( League, @@ -22,10 +38,9 @@ ) from .message import Message from .modifier import ( - GroupedModifierByEffect, + GroupedModifier, Modifier, ModifierCreate, - ModifierInDB, ModifierUpdate, ) from .token import NewPassword, Token, TokenPayload diff --git a/src/backend_api/app/core/schemas/currency.py b/src/backend_api/app/core/schemas/currency.py index 0f09d102..97218af3 100644 --- a/src/backend_api/app/core/schemas/currency.py +++ b/src/backend_api/app/core/schemas/currency.py @@ -2,41 +2,85 @@ # Shared currency props -class _BaseCurrency(_pydantic.BaseModel): +class _BaseCurrencyPrice(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) - tradeName: str - valueInChaos: float + currencyId: int leagueId: int + createdHoursSinceLaunch: int + valueInChaos: float # Properties to receive on currency creation -class CurrencyCreate(_BaseCurrency): - createdHoursSinceLaunch: int +class CurrencyPriceCreate(_BaseCurrencyPrice): + pass # Properties to receive on update -class CurrencyUpdate(_BaseCurrency): +class CurrencyPriceUpdate(_BaseCurrencyPrice): pass # Properties shared by models stored in DB -class CurrencyInDBBase(_BaseCurrency): - createdHoursSinceLaunch: int +class CurrencyPriceInDBBase(_BaseCurrencyPrice): + pass + + +# Properties to return to client +class CurrencyPrice(CurrencyPriceInDBBase): + pass + + +# Properties stored in DB +class CurrencyPriceInDB(CurrencyPriceInDBBase): + pass + + +# Shared currency props +class _BaseCurrencyType(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + + name: str + tradeName: str + + +# Properties to receive on currency creation +class CurrencyTypeCreate(_BaseCurrencyType): + pass + + +# Properties to receive on update +class CurrencyTypeUpdate(_BaseCurrencyType): + currencyId: int + + +# Properties shared by models stored in DB +class CurrencyTypeInDBBase(_BaseCurrencyType): currencyId: int # Properties to return to client -class Currency(CurrencyInDBBase): +class CurrencyType(CurrencyTypeInDBBase): pass # Properties stored in DB -class CurrencyInDB(CurrencyInDBBase): +class CurrencyTypeInDB(CurrencyTypeInDBBase): pass +class Currency(_pydantic.BaseModel): + currencyId: int + tradeName: str + name: str + + leagueId: int + createdHoursSinceLaunch: int + valueInChaos: float + + class CurrencyQuery(_pydantic.BaseModel): createdHoursSinceLaunch: int | None = None + currencyId: int | None = None tradeName: str | None = None leagueId: int | None = None diff --git a/src/backend_api/app/core/schemas/item.py b/src/backend_api/app/core/schemas/item.py index c42c04a3..dd278ab1 100644 --- a/src/backend_api/app/core/schemas/item.py +++ b/src/backend_api/app/core/schemas/item.py @@ -1,4 +1,5 @@ import pydantic as _pydantic +from backend_api.app.core.schemas.item_modifier import ItemModifierCreate class Influences(_pydantic.BaseModel): @@ -11,52 +12,95 @@ class Influences(_pydantic.BaseModel): # Shared item props -class _BaseItem(_pydantic.BaseModel): - model_config = _pydantic.ConfigDict(from_attributes=True) +class ItemBase(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict( + from_attributes=True, extra="ignore", populate_by_name=True + ) - name: str | None = None - leagueId: int - itemBaseTypeId: int + gameItemId: str = _pydantic.Field(alias="game_item_id") + leagueId: int = _pydantic.Field(alias="league_id") + + firstObserved: int = _pydantic.Field(alias="first_observed") + + name: str + itemBaseTypeId: int = _pydantic.Field(alias="item_base_type_id") ilvl: int rarity: str - gameItemId: str | None = None + identified: bool = True - currencyAmount: float | None = None - currencyId: int | None = None corrupted: bool | None = None - delve: bool | None = None + fractured: bool | None = None synthesised: bool | None = None replica: bool | None = None influences: Influences | None = None searing: bool | None = None tangled: bool | None = None - prefixes: int | None = None - suffixes: int | None = None - foilVariation: int | None = None -# Properties to receive on item creation -class ItemCreate(_BaseItem): - createdHoursSinceLaunch: int +class ItemPrice(_pydantic.BaseModel): + currencyId: int + currencyAmount: float + isAsync: bool | None = None + + +class ItemCreate(_pydantic.BaseModel): + item: ItemBase + price: ItemPrice + modifiers: list[ItemModifierCreate] # Properties to receive on update -class ItemUpdate(_BaseItem): +class ItemUpdate(ItemBase): pass -# Properties shared by models stored in DB -class ItemInDBBase(_BaseItem): - createdHoursSinceLaunch: int +# Properties to return to client +class Item(ItemBase): itemId: int -# Properties to return to client -class Item(ItemInDBBase): - pass +class ItemQuery(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict( + extra="ignore", + populate_by_name=True, + ) + gameItemId: str + league: str -# Properties stored in DB -class ItemInDB(ItemInDBBase): - pass + +# Shared item props +class _BaseItemAvailability(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + itemId: int + + currencyId: int + currencyAmount: float + + validFrom: int + validTo: int | None = None + + isAsync: bool | None = None + + +class ItemAvailabilityExpired(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + gameItemId: str + leagueId: int + + validTo: int + + +class ItemAvailabilityUpdated(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True) + gameItemId: str + leagueId: int + + price: ItemPrice + + validFrom: int + + +class ItemAvailability(_BaseItemAvailability): + availabilityId: int diff --git a/src/backend_api/app/core/schemas/item_modifier.py b/src/backend_api/app/core/schemas/item_modifier.py index 953f9a1c..f00b55d6 100644 --- a/src/backend_api/app/core/schemas/item_modifier.py +++ b/src/backend_api/app/core/schemas/item_modifier.py @@ -2,7 +2,7 @@ # Shared item modifier props -class _BaseItemModifier(_pydantic.BaseModel): +class ItemModifier(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) itemId: int @@ -11,26 +11,12 @@ class _BaseItemModifier(_pydantic.BaseModel): roll: float | None = None -# Properties to receive on item modifier creation -class ItemModifierCreate(_BaseItemModifier): - createdHoursSinceLaunch: int - - -# Properties to receive on update -class ItemModifierUpdate(_BaseItemModifier): - pass - - -# Properties shared by models stored in DB -class ItemModifierInDBBase(_BaseItemModifier): - createdHoursSinceLaunch: int - +class ItemModifierRoll(_pydantic.BaseModel): + position: int + roll: float | None = None -# Properties to return to client -class ItemModifier(ItemModifierInDBBase): - pass +class ItemModifierCreate(_pydantic.BaseModel): + modifierId: int -# Properties stored in DB -class ItemModifierInDB(ItemModifierInDBBase): - pass + rolls: list[ItemModifierRoll] diff --git a/src/backend_api/app/core/schemas/league.py b/src/backend_api/app/core/schemas/league.py index a9e4f900..98853d08 100644 --- a/src/backend_api/app/core/schemas/league.py +++ b/src/backend_api/app/core/schemas/league.py @@ -1,5 +1,3 @@ -import datetime as _dt - import pydantic as _pydantic @@ -8,8 +6,8 @@ class _BaseLeague(_pydantic.BaseModel): model_config = _pydantic.ConfigDict(from_attributes=True) name: str - validFrom: _dt.datetime - validTo: _dt.datetime | None = None + validFrom: _pydantic.AwareDatetime + validTo: _pydantic.AwareDatetime | None = None version: float diff --git a/src/backend_api/app/core/schemas/modifier.py b/src/backend_api/app/core/schemas/modifier.py index 75d2d621..9bd86412 100644 --- a/src/backend_api/app/core/schemas/modifier.py +++ b/src/backend_api/app/core/schemas/modifier.py @@ -1,20 +1,18 @@ -import datetime as _dt +import re import pydantic as _pydantic # Shared modifier props -class _BaseModifier(_pydantic.BaseModel): - model_config = _pydantic.ConfigDict(from_attributes=True) +class ModifierBase(_pydantic.BaseModel): + model_config = _pydantic.ConfigDict(from_attributes=True, extra="ignore") - position: int relatedUniques: str | None = None - minRoll: float | None = None - maxRoll: float | None = None - textRolls: str | None = None + static: bool | None = None effect: str - regex: str | None = None + regex: re.Pattern | None = None + implicit: bool | None = None explicit: bool | None = None delve: bool | None = None @@ -25,43 +23,47 @@ class _BaseModifier(_pydantic.BaseModel): enchanted: bool | None = None veiled: bool | None = None + @_pydantic.field_validator("regex", mode="before") + @classmethod + def compile_regex(cls, value: str | None) -> re.Pattern: + if value is None: + return None + elif isinstance(value, str): + return re.compile(value) -class GroupedModifierProperties(_pydantic.BaseModel): - position: list[int] - textRolls: list[str | None] - + return value -class GroupedModifierByEffect(_pydantic.BaseModel): - modifierId: int - effect: str - regex: str - static: bool | None - relatedUniques: str | None - groupedModifierProperties: GroupedModifierProperties + @_pydantic.field_serializer("regex") + def serialize_regex(self, regex: re.Pattern | None) -> str | None: + if regex is None: + return None + elif isinstance(regex, str): + return regex + else: + return regex.pattern -# Properties to receive on modifier creation -class ModifierCreate(_BaseModifier): - pass +class ModifierRoll(_pydantic.BaseModel): + position: int + minRoll: float | None = None + maxRoll: float | None = None + textRolls: list[str] | None = None -# Properties to receive on update -class ModifierUpdate(_BaseModifier): - pass +# Properties to receive on modifier creation +class ModifierCreate(ModifierBase): + rolls: list[ModifierRoll] = _pydantic.Field(default_factory=list) -# Properties shared by models stored in DB -class ModifierInDBBase(_BaseModifier): +# Properties to return to client +class Modifier(ModifierBase): modifierId: int - createdAt: _dt.datetime - updatedAt: _dt.datetime | None = None -# Properties to return to client -class Modifier(ModifierInDBBase): - pass +class GroupedModifier(Modifier): + rolls: list[ModifierRoll] -# Properties stored in DB -class ModifierInDB(ModifierInDBBase): - pass +class ModifierUpdate(Modifier): + rolls: list[ModifierRoll] | None = None + effect: str | None = None diff --git a/src/backend_api/app/core/schemas/plot/input.py b/src/backend_api/app/core/schemas/plot/input.py index 9071528c..05171efd 100644 --- a/src/backend_api/app/core/schemas/plot/input.py +++ b/src/backend_api/app/core/schemas/plot/input.py @@ -1,6 +1,5 @@ import pydantic as _pydantic - -from app.core.schemas.item import Influences +from backend_api.app.core.schemas.item import Influences class ItemSpecs(_pydantic.BaseModel): diff --git a/src/backend_api/app/core/schemas/user.py b/src/backend_api/app/core/schemas/user.py index d31b0d93..adbf7af8 100644 --- a/src/backend_api/app/core/schemas/user.py +++ b/src/backend_api/app/core/schemas/user.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import Annotated +from backend_api.app.core.schemas.wrap_validator import custom_error_msg from pydantic import ( BaseModel, ConfigDict, @@ -10,8 +11,6 @@ StringConstraints, ) -from app.core.schemas.wrap_validator import custom_error_msg - class UsernameValidationError(ValueError): @classmethod @@ -21,7 +20,7 @@ def from_validator_exc( exc: Exception | None, ) -> Exception: return cls( - f"Username must be less than 30 characters and can not contain special symbols or white/empty space.", + "Username must be less than 30 characters and can not contain special symbols or white/empty space.", ) diff --git a/src/backend_api/app/crud/__init__.py b/src/backend_api/app/crud/__init__.py index e0b656aa..09934fc1 100644 --- a/src/backend_api/app/crud/__init__.py +++ b/src/backend_api/app/crud/__init__.py @@ -1,81 +1,30 @@ -from app.core.models.models import Currency as model_Currency -from app.core.models.models import Item as model_Item -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.core.models.models import ItemModifier as model_ItemModifier -from app.core.models.models import League as model_League -from app.core.models.models import Modifier as model_Modifier -from app.core.models.models import UnidentifiedItem as model_UnidentifiedItem -from app.core.schemas.currency import Currency, CurrencyCreate -from app.core.schemas.item import Item, ItemCreate, ItemUpdate -from app.core.schemas.item_base_type import ( +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.core.schemas.item_base_type import ( ItemBaseType, ItemBaseTypeCreate, ItemBaseTypeUpdate, ) -from app.core.schemas.item_modifier import ( - ItemModifier, - ItemModifierCreate, - ItemModifierUpdate, -) -from app.core.schemas.league import ( - League, - LeagueCreate, -) -from app.core.schemas.modifier import Modifier, ModifierCreate -from app.core.schemas.unidentified_item import ( - UnidentifiedItem, - UnidentifiedItemCreate, -) -from app.crud.extensions.crud_currency import CRUDCurrency -from app.crud.extensions.crud_league import CRUDLeague -from app.crud.extensions.crud_modifier import CRUDModifier -from app.crud.extensions.crud_unidentifiedItem import CRUDUnidentifiedItem -from app.crud.user import CRUDUser +from backend_api.app.crud.extensions.crud_currency import CRUDCurrency +from backend_api.app.crud.extensions.crud_item import CRUDItem +from backend_api.app.crud.extensions.crud_league import CRUDLeague +from backend_api.app.crud.extensions.crud_modifier import CRUDModifier +from backend_api.app.crud.extensions.crud_unidentifiedItem import CRUDUnidentifiedItem +from backend_api.app.crud.user import CRUDUser from .base import CRUDBase -CRUD_league = CRUDLeague(model=model_League, schema=League, create_schema=LeagueCreate) - -CRUD_currency = CRUDCurrency( - model=model_Currency, - schema=Currency, - create_schema=CurrencyCreate, -) +CRUD_league = CRUDLeague() +CRUD_currency = CRUDCurrency() CRUD_itemBaseType = CRUDBase[ model_ItemBaseType, ItemBaseType, ItemBaseTypeCreate, ItemBaseTypeUpdate ](model=model_ItemBaseType, schema=ItemBaseType, create_schema=ItemBaseTypeCreate) +CRUD_item = CRUDItem() -CRUD_itemModifier = CRUDBase[ - model_ItemModifier, - ItemModifier, - ItemModifierCreate, - ItemModifierUpdate, -]( - model=model_ItemModifier, - schema=ItemModifier, - create_schema=ItemModifierCreate, -) - -CRUD_item = CRUDBase[ - model_Item, - Item, - ItemCreate, - ItemUpdate, -](model=model_Item, schema=Item, create_schema=ItemCreate) +CRUD_unidentifiedItem = CRUDUnidentifiedItem() -CRUD_unidentifiedItem = CRUDUnidentifiedItem( - model=model_UnidentifiedItem, - schema=UnidentifiedItem, - create_schema=UnidentifiedItemCreate, -) - -CRUD_modifier = CRUDModifier( - model=model_Modifier, - schema=Modifier, - create_schema=ModifierCreate, -) +CRUD_modifier = CRUDModifier() CRUD_user = CRUDUser() diff --git a/src/backend_api/app/crud/base.py b/src/backend_api/app/crud/base.py index f243bcdd..56d4be85 100644 --- a/src/backend_api/app/crud/base.py +++ b/src/backend_api/app/crud/base.py @@ -2,23 +2,22 @@ from itertools import islice from typing import Any, Generic, Literal, TypeVar -from pydantic import BaseModel, TypeAdapter -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session - -# from app.api.params import FilterParams -from app.api.params import FilterParams -from app.exceptions import ( +# from backend_api.app.api.params import FilterParams +from backend_api.app.api.params import FilterParams +from backend_api.app.exceptions import ( ArgValueNotSupportedError, DbObjectDoesNotExistError, DbTooManyItemsDeleteError, ) -from app.exceptions.model_exceptions.db_exception import ( +from backend_api.app.exceptions.model_exceptions.db_exception import ( DbObjectAlreadyExistsError, GeneralDBError, ) -from app.logs.logger import logger -from app.utils.sort_algorithms import sort_with_reference +from backend_api.app.logs.logger import logger +from backend_api.app.utils.sort_algorithms import sort_with_reference +from pydantic import BaseModel, TypeAdapter +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session ModelType = TypeVar("ModelType", bound=Any) SchemaType = TypeVar("SchemaType", bound=Any) @@ -55,15 +54,14 @@ def _sort_objects( self, objs: list[ModelType], sort_key: str | None = None, - sort_method: Literal["asc", "dec"] | None = None, + sort_method: Literal["asc", "dec"] = "asc", ) -> list[ModelType]: """ `sort_key` is the column name to sort on. For example `createdAt`. """ if sort_key is None: return objs - if sort_method is None: - sort_method = "asc" + unsorted_extracted_column = [] for obj in objs: unsorted_extracted_column.append(getattr(obj, sort_key)) diff --git a/src/backend_api/app/crud/extensions/crud_currency.py b/src/backend_api/app/crud/extensions/crud_currency.py index 3a094bb9..a29f372d 100644 --- a/src/backend_api/app/crud/extensions/crud_currency.py +++ b/src/backend_api/app/crud/extensions/crud_currency.py @@ -1,49 +1,68 @@ +from backend_api.app.api.params import FilterParams +from backend_api.app.core.models.models import CurrencyPrice as model_CurrencyPrice +from backend_api.app.core.models.models import CurrencyType as model_CurrencyType +from backend_api.app.core.schemas.currency import ( + Currency, + CurrencyPrice, + CurrencyPriceCreate, + CurrencyPriceUpdate, + CurrencyQuery, + CurrencyType, + CurrencyTypeCreate, + CurrencyTypeUpdate, +) +from backend_api.app.crud.base import CRUDBase from pydantic import TypeAdapter from sqlalchemy import select from sqlalchemy.orm import Session -from sqlalchemy.sql.expression import and_, func, or_ +from sqlalchemy.sql.expression import ColumnElement, and_, func, or_ -from app.core.models.models import Currency as model_Currency -from app.core.schemas.currency import ( - Currency, - CurrencyCreate, - CurrencyQuery, - CurrencyUpdate, -) -from app.crud.base import CRUDBase - - -class CRUDCurrency( - CRUDBase[ - model_Currency, - Currency, - CurrencyCreate, - CurrencyUpdate, - ] -): - async def get_latest_currency_id(self, db: Session) -> int: - stmt = select( - func.max(model_Currency.currencyId).label("latestCurrencyId") - ).limit(1) - db_latest_currency_id = db.execute(stmt).mappings().first() - if db_latest_currency_id is not None: - latest_currency_id = db_latest_currency_id["latestCurrencyId"] - else: - latest_currency_id = 1 +class CRUDCurrency: + def __init__(self): + self.crud_price = CRUDBase[ + model_CurrencyPrice, CurrencyPrice, CurrencyPriceCreate, CurrencyPriceUpdate + ]( + model=model_CurrencyPrice, + schema=CurrencyPrice, + create_schema=CurrencyPriceCreate, + ) + self.crud_type = CRUDBase[ + model_CurrencyType, CurrencyType, CurrencyTypeCreate, CurrencyTypeUpdate + ]( + model=model_CurrencyType, + schema=CurrencyType, + create_schema=CurrencyTypeCreate, + ) + + self.latest_hours_validate = TypeAdapter(dict[int, int]).validate_python + self.currency_list_validate = TypeAdapter(list[Currency]).validate_python + + async def get_prices(self, *args, **kwargs): + return await self.crud_price.get(*args, **kwargs) - validate = TypeAdapter(int).validate_python + async def get_types(self, *args, **kwargs): + return await self.crud_type.get(*args, **kwargs) - return validate(latest_currency_id) + async def create_prices(self, *args, **kwargs): + return await self.crud_price.create(*args, **kwargs) + + async def create_types(self, *args, **kwargs): + return await self.crud_type.create(*args, **kwargs) + + async def update_type(self, *args, **kwargs): + return await self.crud_type.update(*args, **kwargs) def _latest_hours_stmt(self, league_ids: list[int]): return ( select( - model_Currency.leagueId, - func.max(model_Currency.createdHoursSinceLaunch).label("latest_hour"), + model_CurrencyPrice.leagueId, + func.max(model_CurrencyPrice.createdHoursSinceLaunch).label( + "latest_hour" + ), ) - .where(model_Currency.leagueId.in_(league_ids)) - .group_by(model_Currency.leagueId) + .where(model_CurrencyPrice.leagueId.in_(league_ids)) + .group_by(model_CurrencyPrice.leagueId) ) async def get_latest_hours( @@ -54,50 +73,88 @@ async def get_latest_hours( objs = db.execute(stmt).mappings().all() id_hour_map = {obj["leagueId"]: obj["latest_hour"] for obj in objs} - validate = TypeAdapter(dict[int, int]).validate_python - - return validate(id_hour_map) + return self.latest_hours_validate(id_hour_map) async def get_latest_currencies( self, db: Session, league_ids: list[int] ) -> list[Currency]: latest_hours = self._latest_hours_stmt(league_ids).subquery() - stmt = select(model_Currency).join( - latest_hours, - (model_Currency.leagueId == latest_hours.c.leagueId) - & (model_Currency.createdHoursSinceLaunch == latest_hours.c.latest_hour), + stmt = ( + select( + model_CurrencyType.currencyId, + model_CurrencyType.name, + model_CurrencyType.tradeName, + model_CurrencyPrice.leagueId, + model_CurrencyPrice.createdHoursSinceLaunch, + model_CurrencyPrice.valueInChaos, + ) + .join( + latest_hours, + (model_CurrencyPrice.leagueId == latest_hours.c.leagueId) + & ( + model_CurrencyPrice.createdHoursSinceLaunch + == latest_hours.c.latest_hour + ), + ) + .join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, + ) ) - currencies = db.scalars(stmt).all() + currencies = db.execute(stmt).mappings().all() - validate = TypeAdapter(list[Currency]).validate_python - - return validate(currencies) + return self.currency_list_validate(currencies) async def get_currency_from_query( - self, db: Session, query_list: list[CurrencyQuery] + self, + db: Session, + query_list: list[CurrencyQuery] | None = None, + filter_params: FilterParams | None = None, ) -> list[Currency]: - filters = [] - for query in query_list: - sub_filter = [] - if query.createdHoursSinceLaunch is not None: - sub_filter.append( - model_Currency.createdHoursSinceLaunch - == query.createdHoursSinceLaunch - ) - - if query.tradeName is not None: - sub_filter.append(model_Currency.tradeName == query.tradeName) - - if query.leagueId is not None: - sub_filter.append(model_Currency.leagueId == query.leagueId) - - if sub_filter: - filters.append(and_(*sub_filter)) - stmt = select(model_Currency).where(or_(*filters)) - - currencies = db.execute(stmt).scalars().all() - - validate = TypeAdapter(list[Currency]).validate_python + stmt = select( + model_CurrencyType.currencyId, + model_CurrencyType.name, + model_CurrencyType.tradeName, + model_CurrencyPrice.leagueId, + model_CurrencyPrice.createdHoursSinceLaunch, + model_CurrencyPrice.valueInChaos, + ).join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, + ) + if query_list: + filters = list[ColumnElement[bool]]() + for query in query_list: + sub_filter = list[ColumnElement[bool]]() + if query.createdHoursSinceLaunch is not None: + sub_filter.append( + model_CurrencyPrice.createdHoursSinceLaunch + == query.createdHoursSinceLaunch + ) + + if query.tradeName is not None: + sub_filter.append(model_CurrencyType.tradeName == query.tradeName) + + if query.leagueId is not None: + sub_filter.append(model_CurrencyPrice.leagueId == query.leagueId) + + if sub_filter: + filters.append(and_(*sub_filter)) + + stmt = stmt.where(or_(*filters)) + + if filter_params is not None: + if filter_params.skip is not None: + stmt = stmt.offset(filter_params.skip) + if filter_params.limit is not None: + stmt = stmt.limit(filter_params.limit) + + currencies = db.execute(stmt).mappings().all() + + if filter_params is not None: + currencies = self.crud_price._sort_objects( + currencies, filter_params.sort_key, filter_params.sort_method + ) - return validate(currencies) + return self.currency_list_validate(currencies) diff --git a/src/backend_api/app/crud/extensions/crud_item.py b/src/backend_api/app/crud/extensions/crud_item.py new file mode 100644 index 00000000..cd565dca --- /dev/null +++ b/src/backend_api/app/crud/extensions/crud_item.py @@ -0,0 +1,313 @@ +from collections import defaultdict +from collections.abc import Iterable +from typing import Any + +from backend_api.app.core.models.models import Item as model_Item +from backend_api.app.core.models.models import ( + ItemAvailability as model_ItemAvailability, +) +from backend_api.app.core.models.models import ItemModifier as model_ItemModifier +from backend_api.app.core.schemas.item import ( + Item, + ItemAvailability, + ItemAvailabilityExpired, + ItemAvailabilityUpdated, + ItemBase, + ItemCreate, +) +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import ( + Boolean, + Float, + Integer, + String, + column, + delete, + select, + update, + values, +) +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session + + +class CRUDItem: + def __init__(self): + self.crud_base = CRUDBase[model_Item, Item, Any, Any]( + model=model_Item, schema=Item, create_schema=None + ) + + self.item_base_adapter = TypeAdapter(list[ItemBase]) + + self.item_validate = TypeAdapter(list[Item]).validate_python + + self.item_availability_validate = TypeAdapter( + list[ItemAvailability] + ).validate_python + + async def get(self, *args, **kwargs): + return await self.crud_base.get(*args, **kwargs) + + async def _incoming_to_database_map( + self, db: Session, *, incoming: Iterable[tuple[str, int]] + ) -> dict[tuple[str, int], int]: + incoming_items_table = values( + column("game_item_id", String), + column("league_id", Integer), + name="incoming_items", + ).data(incoming) + + incoming_to_database_ids_stmt = select( + model_Item.itemId, + model_Item.gameItemId, + model_Item.leagueId, + ).join( + incoming_items_table, + (model_Item.leagueId == incoming_items_table.c.league_id) + & (model_Item.gameItemId == incoming_items_table.c.game_item_id), + ) + + incoming_items = db.execute(incoming_to_database_ids_stmt).mappings().all() + + return { + (incoming.gameItemId, incoming.leagueId): incoming.itemId + for incoming in incoming_items + } + + async def create_items( + self, db: Session, *, new_items: list[ItemCreate] + ) -> list[Item]: + if not new_items: + return [] + + # Deduplicates update_availability, only caring about the latest price + latest_by_item = { + (item.item.gameItemId, item.item.leagueId): item for item in new_items + } + new_items = list(latest_by_item.values()) + + incoming_to_database = await self._incoming_to_database_map( + db, incoming=latest_by_item.keys() + ) + + truely_new_items = list[ItemCreate]() + existing_items = list[ItemCreate]() + + for item in new_items: + game_item_id = item.item.gameItemId + league_id = item.item.leagueId + + db_item_id = incoming_to_database.get((game_item_id, league_id)) + if db_item_id is None: + truely_new_items.append(item) + else: + existing_items.append(item) + + db_items = list[model_Item]() + for new_item in truely_new_items: + db_items.append(model_Item(**new_item.item.model_dump())) + + db.add_all(db_items) + db.flush() + + db_item_availability = list[model_ItemAvailability]() + + instance_count = defaultdict[tuple[int, int], int](int) + db_item_modifiers = list[model_ItemModifier]() + for db_item, new_item in zip(db_items, truely_new_items, strict=True): + for modifier in new_item.modifiers: + key = (db_item.itemId, modifier.modifierId) + for roll in modifier.rolls: + db_item_modifiers.append( + model_ItemModifier( + itemId=db_item.itemId, + modifierId=modifier.modifierId, + position=roll.position, + instance=instance_count[key], + roll=roll.roll, + ) + ) + # a modifier can appear multiple times on an item (eg. forbidden shako) + instance_count[key] += 1 + + db_item_availability.append( + model_ItemAvailability( + itemId=db_item.itemId, + currencyId=new_item.price.currencyId, + currencyAmount=new_item.price.currencyAmount, + validFrom=new_item.item.firstObserved, + isAsync=new_item.price.isAsync, + ) + ) + + db.add_all(db_item_modifiers) + + for existing_item in existing_items: + game_item_id = existing_item.item.gameItemId + league_id = existing_item.item.leagueId + + db_item_id = incoming_to_database[(game_item_id, league_id)] + + db_item_availability.append( + model_ItemAvailability( + itemId=db_item_id, + currencyId=existing_item.price.currencyId, + currencyAmount=existing_item.price.currencyAmount, + validFrom=existing_item.item.firstObserved, + isAsync=existing_item.price.isAsync, + ) + ) + + db.add_all(db_item_availability) + db.flush() + db.commit() + + return self.item_validate(db_items) + + async def update_availability( + self, db: Session, *, updated_availability: list[ItemAvailabilityUpdated] + ) -> list[ItemAvailability]: + if not updated_availability: + return [] + # Deduplicates update_availability, only caring about the latest price + latest_by_item = { + (item.gameItemId, item.leagueId): item for item in updated_availability + } + updated_availability = list(latest_by_item.values()) + + incoming_to_database = await self._incoming_to_database_map( + db, incoming=latest_by_item.keys() + ) + + # removes items not already in the db + incoming_data = list[tuple[int, int, float, int, bool]]() + for item in updated_availability: + item_id = incoming_to_database.get((item.gameItemId, item.leagueId)) + if item_id is None: + continue + + incoming_data.append( + ( + item_id, + item.price.currencyId, + item.price.currencyAmount, + item.validFrom, + item.price.isAsync, + ) + ) + + if not incoming_data: + return [] + + incoming_table = values( + column("item_id", Integer), + column("currency_id", Integer), + column("currency_amount", Float), + column("valid_from", Integer), + column("is_async", Boolean), + name="incoming", + ).data(incoming_data) + # 1. Delete existing same-hour availability + db.execute( + delete(model_ItemAvailability).where( + model_ItemAvailability.itemId == incoming_table.c.item_id, + model_ItemAvailability.validFrom == incoming_table.c.valid_from, + ) + ) + + # 2. Close current availability + db.execute( + update(model_ItemAvailability) + .where( + model_ItemAvailability.itemId == incoming_table.c.item_id, + model_ItemAvailability.validTo.is_(None), + ) + .values( + validTo=incoming_table.c.valid_from, + ) + ) + + # 3. Insert the new availability + new_availability = ( + db.execute( + insert(model_ItemAvailability).from_select( + [ + "itemId", + "currencyId", + "currencyAmount", + "validFrom", + "isAsync", + ], + select( + incoming_table.c.item_id, + incoming_table.c.currency_id, + incoming_table.c.currency_amount, + incoming_table.c.valid_from, + incoming_table.c.is_async, + ), + ) + ) + .scalars() + .all() + ) + + db.commit() + return self.item_availability_validate(new_availability) + + async def patch_expired_availability( + self, db: Session, *, expired_availability: list[ItemAvailabilityExpired] + ) -> list[ItemAvailability]: + if not expired_availability: + return [] + incoming_to_database = await self._incoming_to_database_map( + db, + incoming=[ + (item.gameItemId, item.leagueId) for item in expired_availability + ], + ) + + # removes items which don't already exist in db + expired_data = list[tuple[str, int]]() + for item in expired_availability: + item_id = incoming_to_database.get((item.gameItemId, item.leagueId)) + if item_id is None: + continue + + expired_data.append((item_id, item.validTo)) + + if not expired_data: + return [] + + expired_table = values( + column("item_id", Integer), + column("valid_to", Integer), + name="expired", + ).data(expired_data) + # 1. Delete existing same-hour availability + db.execute( + delete(model_ItemAvailability).where( + model_ItemAvailability.itemId == expired_table.c.item_id, + model_ItemAvailability.validFrom == expired_table.c.valid_to, + ) + ) + + # 2. Close current availability + updated_availability = ( + db.execute( + update(model_ItemAvailability) + .where( + model_ItemAvailability.itemId == expired_table.c.item_id, + model_ItemAvailability.validTo.is_(None), + ) + .values( + validTo=expired_table.c.valid_to, + ) + ) + .scalars() + .all() + ) + + db.commit() + + return self.item_availability_validate(updated_availability) diff --git a/src/backend_api/app/crud/extensions/crud_league.py b/src/backend_api/app/crud/extensions/crud_league.py index 01af9728..acc2d623 100644 --- a/src/backend_api/app/crud/extensions/crud_league.py +++ b/src/backend_api/app/crud/extensions/crud_league.py @@ -1,19 +1,21 @@ from datetime import datetime -from pydantic import TypeAdapter -from sqlalchemy import select -from sqlalchemy.orm import Session - -from app.core.models.models import League as model_League -from app.core.schemas.league import ( +from backend_api.app.core.models.models import League as model_League +from backend_api.app.core.schemas.league import ( League, LeagueCreate, LeagueUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import select +from sqlalchemy.orm import Session class CRUDLeague(CRUDBase[model_League, League, LeagueCreate, LeagueUpdate]): + def __init__(self): + super().__init__(model=model_League, schema=League, create_schema=LeagueCreate) + async def get_active_leagues(self, db: Session) -> list[League]: now = datetime.now() diff --git a/src/backend_api/app/crud/extensions/crud_modifier.py b/src/backend_api/app/crud/extensions/crud_modifier.py index 9d291ea4..02d3d281 100644 --- a/src/backend_api/app/crud/extensions/crud_modifier.py +++ b/src/backend_api/app/crud/extensions/crud_modifier.py @@ -1,16 +1,16 @@ -from fastapi import HTTPException -from pydantic import TypeAdapter -from sqlalchemy import func, select -from sqlalchemy.orm import Session - -from app.core.models.models import Modifier as model_Modifier -from app.core.schemas.modifier import ( - GroupedModifierByEffect, +from backend_api.app.core.models.models import Modifier as model_Modifier +from backend_api.app.core.models.models import ModifierRoll as model_ModifierRoll +from backend_api.app.core.schemas.modifier import ( + GroupedModifier, Modifier, ModifierCreate, ModifierUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import func, select, update +from sqlalchemy.dialects.postgresql import aggregate_order_by +from sqlalchemy.orm import Session class CRUDModifier( @@ -21,36 +21,92 @@ class CRUDModifier( ModifierUpdate, ] ): - async def get_grouped_modifier_by_effect( - self, db: Session - ) -> GroupedModifierByEffect: - stmt = select( - model_Modifier.modifierId, - func.min(model_Modifier.effect).label("effect"), - func.coalesce( - func.min(model_Modifier.regex), - func.min(model_Modifier.effect), - ).label("regex"), - func.min(model_Modifier.relatedUniques).label("relatedUniques"), - func.bool_or(model_Modifier.static).label("static"), - func.json_build_object( - "position", - func.json_agg(model_Modifier.position), - "textRolls", - func.json_agg(model_Modifier.textRolls), - ).label("groupedModifierProperties"), - ).group_by(model_Modifier.modifierId) - - grouped_modifier_by_effect_record = db.execute(stmt).mappings().all() - - if not grouped_modifier_by_effect_record: - raise HTTPException( - status_code=404, - detail=f"No objects found in the table {self.model.__tablename__}.", + def __init__(self): + super().__init__( + model=model_Modifier, + schema=Modifier, + create_schema=ModifierCreate, + ) + + async def create( + self, db: Session, *, modifiers: list[ModifierCreate] + ) -> list[Modifier]: + db_modifiers = [ + model_Modifier(**modifier.model_dump(exclude="rolls")) + for modifier in modifiers + ] + + db.add_all(db_modifiers) + db.flush() + + db_modifier_rolls = [ + model_ModifierRoll( + modifierId=db_modifier.modifierId, **modifier_roll.model_dump() ) + for db_modifier, modifier in zip(db_modifiers, modifiers, strict=True) + for modifier_roll in modifier.rolls + ] + db.add_all(db_modifier_rolls) + db.flush() + db.commit() - validate = TypeAdapter( - GroupedModifierByEffect | list[GroupedModifierByEffect] - ).validate_python + return self.validate(db_modifiers) + + async def update(self, db: Session, *, modifier: ModifierUpdate): + update_modifier_stmt = ( + update(model_Modifier) + .where(model_Modifier.modifierId == modifier.modifierId) + .values( + **modifier.model_dump( + exclude_unset=True, exclude_none=True, exclude={"rolls"} + ) + ) + ) + db.execute(update_modifier_stmt) + + if modifier.rolls is not None: + modifier_rolls = [ + { + "modifierId": modifier.modifierId, + **roll.model_dump(exclude_unset=True), + } + for roll in modifier.rolls + ] + db.execute(update(model_ModifierRoll), modifier_rolls) + + db.commit() + + async def get_grouped_modifiers(self, db: Session) -> list[GroupedModifier]: + grouped_modifier_rolls = ( + select( + model_ModifierRoll.modifierId, + func.json_agg( + aggregate_order_by( + func.json_build_object( + "position", + model_ModifierRoll.position, + "minRoll", + model_ModifierRoll.minRoll, + "maxRoll", + model_ModifierRoll.maxRoll, + "textRolls", + model_ModifierRoll.textRolls, + ), + model_ModifierRoll.modifierId, + model_ModifierRoll.position, + ), + ).label("rolls"), + ) + .group_by(model_ModifierRoll.modifierId) + .cte("grouped_modifier_rolls") + ) + + stmt = select( + *model_Modifier.__table__.columns, grouped_modifier_rolls.c.rolls + ).join( + grouped_modifier_rolls, + model_Modifier.modifierId == grouped_modifier_rolls.c.modifierId, + ) + grouped_modifiers = db.execute(stmt).mappings().all() - return validate(grouped_modifier_by_effect_record) + return TypeAdapter(list[GroupedModifier]).validate_python(grouped_modifiers) diff --git a/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py b/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py index 3e3ad12e..ca9b578b 100644 --- a/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py +++ b/src/backend_api/app/crud/extensions/crud_unidentifiedItem.py @@ -1,14 +1,15 @@ -from pydantic import TypeAdapter -from sqlalchemy import delete, select -from sqlalchemy.orm import Session - -from app.core.models.models import UnidentifiedItem as model_UnidentifiedItem -from app.core.schemas.unidentified_item import ( +from backend_api.app.core.models.models import ( + UnidentifiedItem as model_UnidentifiedItem, +) +from backend_api.app.core.schemas.unidentified_item import ( UnidentifiedItem, UnidentifiedItemCreate, UnidentifiedItemUpdate, ) -from app.crud.base import CRUDBase +from backend_api.app.crud.base import CRUDBase +from pydantic import TypeAdapter +from sqlalchemy import delete, select +from sqlalchemy.orm import Session class CRUDUnidentifiedItem( @@ -19,6 +20,13 @@ class CRUDUnidentifiedItem( UnidentifiedItemUpdate, ] ): + def __init__(self): + super().__init__( + model=model_UnidentifiedItem, + schema=UnidentifiedItem, + create_schema=UnidentifiedItemCreate, + ) + async def get_non_aggregated(self, db: Session) -> list[UnidentifiedItem]: """ Returns the non aggregated unidentified items diff --git a/src/backend_api/app/crud/user.py b/src/backend_api/app/crud/user.py index ad60c0a1..43424d11 100644 --- a/src/backend_api/app/crud/user.py +++ b/src/backend_api/app/crud/user.py @@ -1,19 +1,18 @@ from uuid import UUID -from pydantic import EmailStr, TypeAdapter -from sqlalchemy.orm import Session -from sqlalchemy.sql import func, select - -from app.core.models.models import User as model_User -from app.core.schemas import User, UserCreate, UserUpdate -from app.core.schemas.user import UpdatePassword, UsersPublic -from app.core.security import get_password_hash, verify_password -from app.exceptions import ( +from backend_api.app.core.models.models import User as model_User +from backend_api.app.core.schemas import User, UserCreate, UserUpdate +from backend_api.app.core.schemas.user import UpdatePassword, UsersPublic +from backend_api.app.core.security import get_password_hash, verify_password +from backend_api.app.exceptions import ( DbObjectAlreadyExistsError, DbObjectDoesNotExistError, InvalidPasswordError, NewPasswordIsSameError, ) +from pydantic import EmailStr, TypeAdapter +from sqlalchemy.orm import Session +from sqlalchemy.sql import func, select class CRUDUser: diff --git a/src/backend_api/app/exception_handlers.py b/src/backend_api/app/exception_handlers.py index 6af23e43..a7f50510 100644 --- a/src/backend_api/app/exception_handlers.py +++ b/src/backend_api/app/exception_handlers.py @@ -1,5 +1,9 @@ import sys +from backend_api.app.exceptions.model_exceptions.rate_limit_exception import ( + RateLimitExceededError, +) +from backend_api.app.logs.logger import logger from fastapi import Request from fastapi.exception_handlers import http_exception_handler as _http_exception_handler from fastapi.exception_handlers import ( @@ -9,9 +13,6 @@ from fastapi.responses import JSONResponse, PlainTextResponse, Response from slowapi.errors import RateLimitExceeded -from app.exceptions.model_exceptions.rate_limit_exception import RateLimitExceededError -from app.logs.logger import logger - """ Taken from https://medium.com/@roy-pstr/fastapi-server-errors-and-logs-take-back-control-696405437983 diff --git a/src/backend_api/app/exceptions/model_exceptions/cache_exception.py b/src/backend_api/app/exceptions/model_exceptions/cache_exception.py index d3c76342..a4128243 100644 --- a/src/backend_api/app/exceptions/model_exceptions/cache_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/cache_exception.py @@ -1,6 +1,5 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class InvalidCacheUpdateParamsError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/db_exception.py b/src/backend_api/app/exceptions/model_exceptions/db_exception.py index c83119ae..b26e209d 100644 --- a/src/backend_api/app/exceptions/model_exceptions/db_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/db_exception.py @@ -1,10 +1,9 @@ from typing import Any import starlette.status as status - -from app.core.models.models import User as model_User -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import logger +from backend_api.app.core.models.models import User as model_User +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import logger HIDDEN_TABLE_LIST = [model_User.__tablename__] diff --git a/src/backend_api/app/exceptions/model_exceptions/plot_exception.py b/src/backend_api/app/exceptions/model_exceptions/plot_exception.py index a3d02943..0c9a24ed 100644 --- a/src/backend_api/app/exceptions/model_exceptions/plot_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/plot_exception.py @@ -1,7 +1,6 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import logger +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import logger class PlotQueryToDBError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py b/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py index 75ae9a4b..5cc9f478 100644 --- a/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/rate_limit_exception.py @@ -1,6 +1,5 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class RateLimitExceededError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/request_exception.py b/src/backend_api/app/exceptions/model_exceptions/request_exception.py index 89fa1cc3..19dbc2e3 100644 --- a/src/backend_api/app/exceptions/model_exceptions/request_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/request_exception.py @@ -1,8 +1,7 @@ from typing import Any import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class InvalidTokenError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/test_exception.py b/src/backend_api/app/exceptions/model_exceptions/test_exception.py index 5dbd8add..b037495d 100644 --- a/src/backend_api/app/exceptions/model_exceptions/test_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/test_exception.py @@ -1,7 +1,6 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import test_logger +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import test_logger class _TestErrorLogBase(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py b/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py index 1a1d9869..0b008af5 100644 --- a/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/user_login_exception.py @@ -1,6 +1,5 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError class UserWithNotEnoughPrivilegesError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/exceptions/model_exceptions/util_exception.py b/src/backend_api/app/exceptions/model_exceptions/util_exception.py index d8c6715f..0695ebfe 100644 --- a/src/backend_api/app/exceptions/model_exceptions/util_exception.py +++ b/src/backend_api/app/exceptions/model_exceptions/util_exception.py @@ -1,7 +1,6 @@ import starlette.status as status - -from app.exceptions.exception_base import PathOfModifiersAPIError -from app.logs.logger import logger +from backend_api.app.exceptions.exception_base import PathOfModifiersAPIError +from backend_api.app.logs.logger import logger class _UtilBaseError(PathOfModifiersAPIError): diff --git a/src/backend_api/app/initial_data.py b/src/backend_api/app/initial_data.py index de228f1c..9e6a2eb8 100644 --- a/src/backend_api/app/initial_data.py +++ b/src/backend_api/app/initial_data.py @@ -1,10 +1,9 @@ import logging +from backend_api.app.core.models.database import engine +from backend_api.app.core.models.init_db import init_db from sqlalchemy.orm import Session -from app.core.models.database import engine -from app.core.models.init_db import init_db - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/backend_api/app/logs/config/__init__.py b/src/backend_api/app/logs/config/__init__.py index 91bf8f59..aa521277 100644 --- a/src/backend_api/app/logs/config/__init__.py +++ b/src/backend_api/app/logs/config/__init__.py @@ -1,2 +1,5 @@ -from app.logs.config.formatters import JSONFormatter, APIMessageFormatter -from app.logs.config.data_filter import SensitiveDataFilter, UnwantedDataFilter +from backend_api.app.logs.config.formatters import JSONFormatter, APIMessageFormatter +from backend_api.app.logs.config.data_filter import ( + SensitiveDataFilter, + UnwantedDataFilter, +) diff --git a/src/backend_api/app/logs/config/config.yml b/src/backend_api/app/logs/config/config.yml index a81e1f6b..cbc5abbb 100644 --- a/src/backend_api/app/logs/config/config.yml +++ b/src/backend_api/app/logs/config/config.yml @@ -5,7 +5,7 @@ formatters: format: "[%(name)s|%(levelname)s|%(module)s]: %(asctime)s: %(message)s" datefmt: "%Y-%m-%dT%H:%M:%S%z" json: - (): app.logs.config.JSONFormatter + (): backend_api.app.logs.config.JSONFormatter fmt_keys: logger: name level: levelname @@ -13,9 +13,9 @@ formatters: message: message filters: sensitive_data_filter: - (): app.logs.config.SensitiveDataFilter + (): backend_api.app.logs.config.SensitiveDataFilter unwanted_data_filter: - (): app.logs.config.UnwantedDataFilter + (): backend_api.app.logs.config.UnwantedDataFilter handlers: stdout: class: logging.StreamHandler @@ -34,7 +34,7 @@ handlers: level: DEBUG filters: [sensitive_data_filter] formatter: simple - filename: "app/logs/backend-api.log" + filename: "backend_api/app/logs/backend-api.log" maxBytes: 504857600 # 500 MB backupCount: 2 loggers: diff --git a/src/backend_api/app/main.py b/src/backend_api/app/main.py index 908dc17c..1d260c12 100644 --- a/src/backend_api/app/main.py +++ b/src/backend_api/app/main.py @@ -1,5 +1,20 @@ from contextlib import asynccontextmanager +from backend_api.app.api.api import api_router +from backend_api.app.core.config import settings +from backend_api.app.core.models.database import async_engine +from backend_api.app.exception_handlers import ( + custom_rate_limit_exceeded_handler, + http_exception_handler, + request_validation_exception_handler, + slow_api_rate_limit_exceeded_handler, + unhandled_exception_handler, +) +from backend_api.app.exceptions.model_exceptions.rate_limit_exception import ( + RateLimitExceededError, +) +from backend_api.app.logs.logger import setup_logging +from backend_api.app.middleware.request_logs import log_request_middleware from fastapi import FastAPI from fastapi.exceptions import RequestValidationError from fastapi.routing import APIRoute @@ -8,20 +23,6 @@ from starlette.middleware.cors import CORSMiddleware from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware -from app.api.api import api_router -from app.core.config import settings -from app.core.models.database import async_engine -from app.exception_handlers import ( - custom_rate_limit_exceeded_handler, - http_exception_handler, - request_validation_exception_handler, - slow_api_rate_limit_exceeded_handler, - unhandled_exception_handler, -) -from app.exceptions.model_exceptions.rate_limit_exception import RateLimitExceededError -from app.logs.logger import setup_logging -from app.middleware.request_logs import log_request_middleware - def custom_generate_unique_id(route: APIRoute) -> str: return f"{route.tags[0]}-{route.name}" diff --git a/src/backend_api/app/middleware/request_logs.py b/src/backend_api/app/middleware/request_logs.py index 53fd22c8..276dd7ef 100644 --- a/src/backend_api/app/middleware/request_logs.py +++ b/src/backend_api/app/middleware/request_logs.py @@ -1,12 +1,11 @@ import http import time -from fastapi import Request - -from app.api.deps import ( +from backend_api.app.api.deps import ( get_user_ip_from_header, ) -from app.logs.logger import logger_request +from backend_api.app.logs.logger import logger_request +from fastapi import Request """ Taken from https://medium.com/@roy-pstr/fastapi-server-errors-and-logs-take-back-control-696405437983 diff --git a/src/backend_api/app/plotting/plotter.py b/src/backend_api/app/plotting/plotter.py index 013a88db..8bfe3dc7 100644 --- a/src/backend_api/app/plotting/plotter.py +++ b/src/backend_api/app/plotting/plotter.py @@ -2,44 +2,29 @@ from typing import Any, Generic, TypeVar import pandas as pd -from pydantic import TypeAdapter -from pydantic.fields import FieldInfo -from sqlalchemy import ( - BinaryExpression, - ColumnElement, - Label, - Result, - and_, - case, - desc, - func, - or_, - select, +from backend_api.app.core.models.database import engine +from backend_api.app.core.models.models import ( + CurrencyPrice as model_CurrencyPrice, ) -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import InstrumentedAttribute -from sqlalchemy.sql.expression import Select - -from app.core.models.database import engine -from app.core.models.models import ( - Currency as model_Currency, +from backend_api.app.core.models.models import ( + CurrencyType as model_CurrencyType, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Item as model_Item, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( ItemBaseType as model_ItemBaseType, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( ItemModifier as model_ItemModifier, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( UnidentifiedItem as model_UniItem, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( UnidentifiedItem as model_UnidentifiedItem, ) -from app.core.schemas.plot import ( +from backend_api.app.core.schemas.plot import ( BasePlotQuery, IdentifiedPlotQuery, ItemSpecs, @@ -49,12 +34,32 @@ UnidentifiedPlotQuery, WantedModifier, ) -from app.exceptions.model_exceptions.plot_exception import ( +from backend_api.app.exceptions.model_exceptions.plot_exception import ( PlotQueryDataNotFoundError, PlotQueryInvalidError, ) -from app.logs.logger import plot_logger -from app.utils.timing_tracker import async_timing_tracker, sync_timing_tracker +from backend_api.app.logs.logger import plot_logger +from backend_api.app.utils.timing_tracker import ( + async_timing_tracker, + sync_timing_tracker, +) +from pydantic import TypeAdapter +from pydantic.fields import FieldInfo +from sqlalchemy import ( + BinaryExpression, + ColumnElement, + Label, + Result, + and_, + case, + desc, + func, + or_, + select, +) +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.sql.expression import Select Q = TypeVar("Q", bound=PlotQuery) @@ -101,17 +106,25 @@ def _init_stmt( item_model.itemBaseTypeId, item_model.currencyId, item_model.currencyAmount, - model_Currency.tradeName, - model_Currency.valueInChaos, - model_Currency.createdHoursSinceLaunch.label( + model_CurrencyType.tradeName, + model_CurrencyPrice.valueInChaos, + model_CurrencyPrice.createdHoursSinceLaunch.label( "currencyCreatedHoursSinceLaunch" ), ] if query_select_args: select_args.extend(query_select_args) - stmt = select(*select_args).join( - model_Currency, item_model.currencyId == model_Currency.currencyId + stmt = ( + select(*select_args) + .join( + model_CurrencyPrice, + item_model.currencyId == model_CurrencyPrice.currencyId, + ) + .join( + model_CurrencyType, + model_CurrencyPrice.currencyId == model_CurrencyType.currencyId, + ) ) if isinstance(query.leagueId, list): diff --git a/src/backend_api/app/tests/test_real_env/api/real_env_base.py b/src/backend_api/app/tests/test_real_env/api/real_env_base.py index 8450d5c9..76c3773e 100644 --- a/src/backend_api/app/tests/test_real_env/api/real_env_base.py +++ b/src/backend_api/app/tests/test_real_env/api/real_env_base.py @@ -1,13 +1,12 @@ from collections.abc import AsyncGenerator +from backend_api.app.core.config import settings +from backend_api.app.logs.logger import setup_logging +from backend_api.app.logs.logger import test_logger as logger +from backend_api.app.main import app +from backend_api.app.tests.utils.utils import get_superuser_token_headers from httpx import AsyncClient -from app.core.config import settings -from app.logs.logger import setup_logging -from app.logs.logger import test_logger as logger -from app.main import app -from app.tests.utils.utils import get_superuser_token_headers - class RealEnvBase: """Inheritable test real env base class with help functions""" diff --git a/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py b/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py index 7e8bcf8a..9103df5a 100644 --- a/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py +++ b/src/backend_api/app/tests/test_real_env/api/real_env_cases/plot.py @@ -3,8 +3,8 @@ from json import JSONDecodeError from typing import Any -from app.logs.logger import test_logger as logger -from app.tests.test_real_env.api.real_env_base import RealEnvBase +from backend_api.app.logs.logger import test_logger as logger +from backend_api.app.tests.test_real_env.api.real_env_base import RealEnvBase class TestPlotRealEnv(RealEnvBase): diff --git a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py index 860da99d..1e3f18fa 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py +++ b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_base.py @@ -3,22 +3,25 @@ from typing import Any import pytest -from httpx import AsyncClient, Response -from sqlalchemy.orm import Session - -from app.api.api_message_util import ( +from backend_api.app.api.api_message_util import ( get_delete_return_msg, ) -from app.core.cache.user_cache import UserCache -from app.core.config import settings -from app.crud.base import CRUDBase, ModelType -from app.exceptions import ( +from backend_api.app.core.cache.user_cache import UserCache +from backend_api.app.core.config import settings +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.exceptions import ( DbObjectDoesNotExistError, ) -from app.exceptions.model_exceptions.db_exception import DbObjectAlreadyExistsError -from app.exceptions.model_exceptions.request_exception import InvalidTokenError -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import is_courotine_function +from backend_api.app.exceptions.model_exceptions.db_exception import ( + DbObjectAlreadyExistsError, +) +from backend_api.app.exceptions.model_exceptions.request_exception import ( + InvalidTokenError, +) +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import is_courotine_function +from httpx import AsyncClient, Response +from sqlalchemy.orm import Session @pytest.mark.usefixtures("clear_db", autouse=True) diff --git a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py index 74e1a5a7..7820a406 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py +++ b/src/backend_api/app/tests/test_simulating_env/api/api_routes_test_slowapi_rate_limit.py @@ -2,13 +2,14 @@ from typing import Any import pytest +from backend_api.app.core.config import settings +from backend_api.app.tests.test_simulating_env.api.api_test_rate_limit_base import ( + TestRateLimitBase, +) +from backend_api.app.tests.utils.rate_limit import RateLimitPerTimeInterval from slowapi import Limiter from sqlalchemy.orm import Session -from app.core.config import settings -from app.tests.test_simulating_env.api.api_test_rate_limit_base import TestRateLimitBase -from app.tests.utils.rate_limit import RateLimitPerTimeInterval - @pytest.mark.usefixtures("clear_db", autouse=True) @pytest.mark.skipif( diff --git a/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py b/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py index 8cab5d94..4080dabe 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py +++ b/src/backend_api/app/tests/test_simulating_env/api/api_test_rate_limit_base.py @@ -3,8 +3,8 @@ from typing import Any from unittest.mock import patch -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.rate_limit import RateLimitPerTimeInterval +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.rate_limit import RateLimitPerTimeInterval class TestRateLimitBase(BaseTest): diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py index 5ed1a9df..54e80637 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_currency.py @@ -3,28 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import currency_prefix, league_prefix -from app.api.routes.currency import get_currency -from app.core.config import settings -from app.core.models.models import Currency, League -from app.crud import CRUD_currency -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.currency import ( +from backend_api.app.api.routes import currency_prefix, league_prefix +from backend_api.app.api.routes.currency import get_currency +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Currency, League +from backend_api.app.crud import CRUD_currency +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.currency import ( create_random_currency_dict, generate_random_currency, ) -from app.tests.utils.rate_limit import ( +from backend_api.app.tests.utils.rate_limit import ( RateLimitPerTimeInterval, get_function_decorator_rate_limit_per_time_interval, ) -from app.tests.utils.utils import ( +from backend_api.app.tests.utils.utils import ( get_model_table_name, get_model_unique_identifier, ) +from fastapi import Response +from httpx import AsyncClient + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py index e0170825..bd7fffde 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item.py @@ -3,25 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient -from sqlalchemy.orm import Session - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import ( +from backend_api.app.api.routes import ( currency_prefix, item_base_type_prefix, item_prefix, ) -from app.core.config import settings -from app.core.models.models import Currency, Item, ItemBaseType -from app.crud import CRUD_item -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item import ( +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Currency, Item, ItemBaseType +from backend_api.app.crud import CRUD_item +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item import ( create_random_item_dict, generate_random_item, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from fastapi import Response +from httpx import AsyncClient +from sqlalchemy.orm import Session + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py index 7adb4a17..b02c677c 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_base_type.py @@ -3,25 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import item_base_type_prefix -from app.api.routes.item_base_type import get_item_base_type -from app.core.config import settings -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.crud import CRUD_itemBaseType -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item_base_type import ( +from backend_api.app.api.routes import item_base_type_prefix +from backend_api.app.api.routes.item_base_type import get_item_base_type +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.crud import CRUD_itemBaseType +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item_base_type import ( create_random_item_base_type_dict, generate_random_item_base_type, ) -from app.tests.utils.rate_limit import ( +from backend_api.app.tests.utils.rate_limit import ( RateLimitPerTimeInterval, get_function_decorator_rate_limit_per_time_interval, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from fastapi import Response +from httpx import AsyncClient + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py index e695dfe2..26eeb702 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_item_modifier.py @@ -1,29 +1,31 @@ from collections.abc import Awaitable, Callable import pytest -from sqlalchemy.orm import Session - -from app.api.routes import ( +from backend_api.app.api.routes import ( currency_prefix, item_base_type_prefix, item_modifier_prefix, item_prefix, modifier_prefix, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Currency, Item, ItemBaseType, ItemModifier, Modifier, ) -from app.crud import CRUD_itemModifier -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item_modifier import ( +from backend_api.app.crud import CRUD_itemModifier +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item_modifier import ( create_random_item_modifier_dict, generate_random_item_modifier, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from sqlalchemy.orm import Session @pytest.fixture(scope="module") @@ -114,14 +116,16 @@ async def create_object(db: Session) -> dict: @pytest.fixture(scope="module") -def object_generator_func_w_deps() -> Callable[ - [], - tuple[ - dict, - ItemModifier, - list[dict | Item | ItemBaseType | Currency | Modifier], - ], -]: +def object_generator_func_w_deps() -> ( + Callable[ + [], + tuple[ + dict, + ItemModifier, + list[dict | Item | ItemBaseType | Currency | Modifier], + ], + ] +): def generate_random_item_modifier_w_deps( db, ) -> Callable[ diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py index 249220af..d8336251 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_league.py @@ -3,21 +3,24 @@ import pytest import pytest_asyncio +from backend_api.app.api.routes import item_prefix +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import League +from backend_api.app.crud import CRUD_item +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.item import ( + create_random_item_dict, + generate_random_item, +) +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) from fastapi import Response from httpx import AsyncClient from sqlalchemy.orm import Session import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import item_prefix -from app.core.config import settings -from app.core.models.models import League -from app.crud import CRUD_item -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.item import ( - create_random_item_dict, - generate_random_item, -) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py index 34afc785..db039b82 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_login.py @@ -1,16 +1,17 @@ from collections.abc import Awaitable import pytest -from fastapi import Response -from httpx import AsyncClient - -from app.api.routes.login import login_access_session, login_prefix -from app.core.config import settings -from app.tests.test_simulating_env.api.api_test_rate_limit_base import TestRateLimitBase -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.rate_limit import ( +from backend_api.app.api.routes.login import login_access_session, login_prefix +from backend_api.app.core.config import settings +from backend_api.app.tests.test_simulating_env.api.api_test_rate_limit_base import ( + TestRateLimitBase, +) +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.rate_limit import ( get_function_decorator_rate_limit_per_time_interval, ) +from fastapi import Response +from httpx import AsyncClient @pytest.mark.usefixtures("clear_db", autouse=True) diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py index 1c51af94..61888dc8 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_modifier.py @@ -3,25 +3,28 @@ import pytest import pytest_asyncio -from fastapi import Response -from httpx import AsyncClient - -import app.tests.test_simulating_env.api.api_routes_test_base as test_api -from app.api.routes import modifier_prefix -from app.api.routes.modifier import get_modifier -from app.core.config import settings -from app.core.models.models import Modifier -from app.crud import CRUD_modifier -from app.crud.base import CRUDBase, ModelType -from app.tests.utils.model_utils.modifier import ( +from backend_api.app.api.routes import modifier_prefix +from backend_api.app.api.routes.modifier import get_modifier +from backend_api.app.core.config import settings +from backend_api.app.core.models.models import Modifier +from backend_api.app.crud import CRUD_modifier +from backend_api.app.crud.base import CRUDBase, ModelType +from backend_api.app.tests.utils.model_utils.modifier import ( create_random_modifier_dict, generate_random_modifier, ) -from app.tests.utils.rate_limit import ( +from backend_api.app.tests.utils.rate_limit import ( RateLimitPerTimeInterval, get_function_decorator_rate_limit_per_time_interval, ) -from app.tests.utils.utils import get_model_table_name, get_model_unique_identifier +from backend_api.app.tests.utils.utils import ( + get_model_table_name, + get_model_unique_identifier, +) +from fastapi import Response +from httpx import AsyncClient + +import app.tests.test_simulating_env.api.api_routes_test_base as test_api @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py index b06f2243..516cf9bb 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_plot.py @@ -5,17 +5,20 @@ from typing import Any import pytest +from backend_api.app.api.routes.plot import plot_prefix +from backend_api.app.core.config import settings +from backend_api.app.core.rate_limit.rate_limit_config import rate_limit_settings +from backend_api.app.tests.test_simulating_env.api.api_test_rate_limit_base import ( + TestRateLimitBase, +) +from backend_api.app.tests.utils.model_utils.plot import ( + create_minimal_random_plot_query_dict, +) +from backend_api.app.tests.utils.rate_limit import RateLimitPerTimeInterval from fastapi import Response from httpx import AsyncClient from sqlalchemy.orm import Session -from app.api.routes.plot import plot_prefix -from app.core.config import settings -from app.core.rate_limit.rate_limit_config import rate_limit_settings -from app.tests.test_simulating_env.api.api_test_rate_limit_base import TestRateLimitBase -from app.tests.utils.model_utils.plot import create_minimal_random_plot_query_dict -from app.tests.utils.rate_limit import RateLimitPerTimeInterval - @pytest.mark.usefixtures("clear_db", autouse=True) class TestPlotAPI(TestRateLimitBase): diff --git a/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py b/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py index 286a704f..e54213ce 100644 --- a/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py +++ b/src/backend_api/app/tests/test_simulating_env/api/routes/test_turnstile.py @@ -2,14 +2,15 @@ from unittest.mock import patch import pytest +from backend_api.app.api.api_message_util import ( + get_failed_send_challenge_request_error_msg, +) +from backend_api.app.api.routes import turnstile_prefix +from backend_api.app.core.config import settings +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import create_random_ip from httpx import AsyncClient -from app.api.api_message_util import get_failed_send_challenge_request_error_msg -from app.api.routes import turnstile_prefix -from app.core.config import settings -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import create_random_ip - @pytest.mark.usefixtures("clear_db", autouse=True) class TestTurnstileAPI(BaseTest): diff --git a/src/backend_api/app/tests/test_simulating_env/base_test.py b/src/backend_api/app/tests/test_simulating_env/base_test.py index 8bd46ca2..8d3f4fa8 100644 --- a/src/backend_api/app/tests/test_simulating_env/base_test.py +++ b/src/backend_api/app/tests/test_simulating_env/base_test.py @@ -3,15 +3,14 @@ from collections.abc import Awaitable, Callable from typing import Any -from sqlalchemy.inspection import inspect -from sqlalchemy.orm import Session - -from app.core.models.database import insp -from app.crud.base import ( +from backend_api.app.core.models.database import insp +from backend_api.app.crud.base import ( CRUDBase, ModelType, ) -from app.tests.utils.utils import get_extract_functions +from backend_api.app.tests.utils.utils import get_extract_functions +from sqlalchemy.inspection import inspect +from sqlalchemy.orm import Session class BaseTest: diff --git a/src/backend_api/app/tests/test_simulating_env/conftest.py b/src/backend_api/app/tests/test_simulating_env/conftest.py index 01386908..0b9695b5 100644 --- a/src/backend_api/app/tests/test_simulating_env/conftest.py +++ b/src/backend_api/app/tests/test_simulating_env/conftest.py @@ -6,6 +6,23 @@ import pytest import pytest_asyncio +from backend_api.app.api.deps import get_async_db, get_db +from backend_api.app.core.cache.cache import cache +from backend_api.app.core.cache.user_cache import UserCache, UserCacheTokenType +from backend_api.app.core.config import settings +from backend_api.app.core.models.init_db import init_db +from backend_api.app.core.rate_limit.rate_limiters import limiter_ip, limiter_user +from backend_api.app.main import app as actual_app +from backend_api.app.tests.test_simulating_env.setup_test_database import ( + ASYNC_TEST_DATABASE_URL, + TEST_DATABASE_URL, +) +from backend_api.app.tests.utils.database_utils import ( + clear_all_tables, + mock_src_database_for_test_db, +) +from backend_api.app.tests.utils.user import authentication_token_from_email +from backend_api.app.tests.utils.utils import get_superuser_token_headers from fastapi import FastAPI from fastapi.testclient import TestClient from httpx import ASGITransport, AsyncClient @@ -19,24 +36,6 @@ ) from sqlalchemy.orm import Session -from app.api.deps import get_async_db, get_db -from app.core.cache.cache import cache -from app.core.cache.user_cache import UserCache, UserCacheTokenType -from app.core.config import settings -from app.core.models.init_db import init_db -from app.core.rate_limit.rate_limiters import limiter_ip, limiter_user -from app.main import app as actual_app -from app.tests.test_simulating_env.setup_test_database import ( - ASYNC_TEST_DATABASE_URL, - TEST_DATABASE_URL, -) -from app.tests.utils.database_utils import ( - clear_all_tables, - mock_src_database_for_test_db, -) -from app.tests.utils.user import authentication_token_from_email -from app.tests.utils.utils import get_superuser_token_headers - @pytest.fixture(autouse=True) def app() -> Generator[FastAPI, None, None]: diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py index 56b3bb7b..7899f092 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_currency.py @@ -1,13 +1,13 @@ from collections.abc import Callable, Generator import pytest +from backend_api.app.core.models.database import engine +from backend_api.app.crud import CRUD_currency +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.currency import generate_random_currency from sqlalchemy.orm import Session import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.database import engine -from app.crud import CRUD_currency -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.currency import generate_random_currency @pytest.fixture(scope="session") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py index 14dc94f6..18f196fd 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item.py @@ -1,16 +1,16 @@ from collections.abc import Callable import pytest - -import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.models import Currency, Item, ItemBaseType -from app.crud import ( +from backend_api.app.core.models.models import Currency, Item, ItemBaseType +from backend_api.app.crud import ( CRUD_currency, CRUD_item, CRUD_itemBaseType, ) -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.item import generate_random_item +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.item import generate_random_item + +import app.tests.test_simulating_env.crud.crud_test_base as test_crud @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py index f7c47c46..4c941b39 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_base_type.py @@ -1,12 +1,14 @@ from collections.abc import Callable import pytest +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.crud import CRUD_itemBaseType +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.item_base_type import ( + generate_random_item_base_type, +) import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.crud import CRUD_itemBaseType -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.item_base_type import generate_random_item_base_type @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py index 7e22cf35..ad20344b 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_item_modifier.py @@ -1,23 +1,24 @@ from collections.abc import Callable import pytest - -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Currency, Item, ItemBaseType, ItemModifier, Modifier, ) -from app.crud import ( +from backend_api.app.crud import ( CRUD_currency, CRUD_item, CRUD_itemBaseType, CRUD_itemModifier, CRUD_modifier, ) -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.item_modifier import generate_random_item_modifier +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.item_modifier import ( + generate_random_item_modifier, +) @pytest.fixture(scope="module") @@ -31,14 +32,16 @@ def is_hypertable() -> bool: @pytest.fixture(scope="module") -def object_generator_func_w_deps() -> Callable[ - [], - tuple[ - dict, - ItemModifier, - list[dict | Item | ItemBaseType | Currency | Modifier], - ], -]: +def object_generator_func_w_deps() -> ( + Callable[ + [], + tuple[ + dict, + ItemModifier, + list[dict | Item | ItemBaseType | Currency | Modifier], + ], + ] +): def generate_random_item_modifier_w_deps( db, ) -> Callable[ diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py index e5dccb9c..28031cf6 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_league.py @@ -1,13 +1,13 @@ from collections.abc import Callable, Generator import pytest +from backend_api.app.core.models.database import engine +from backend_api.app.crud import CRUD_league +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.league import generate_random_league from sqlalchemy.orm import Session import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.core.models.database import engine -from app.crud import CRUD_league -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.league import generate_random_league @pytest.fixture(scope="session") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py index 3bbc724c..9933bf66 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_modifier.py @@ -1,9 +1,9 @@ import pytest +from backend_api.app.crud import CRUD_modifier +from backend_api.app.crud.base import CRUDBase +from backend_api.app.tests.utils.model_utils.modifier import generate_random_modifier import app.tests.test_simulating_env.crud.crud_test_base as test_crud -from app.crud import CRUD_modifier -from app.crud.base import CRUDBase -from app.tests.utils.model_utils.modifier import generate_random_modifier @pytest.fixture(scope="module") diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py index 430268d2..a0bce5d5 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_models/test_user.py @@ -1,4 +1,4 @@ -from app.tests.test_simulating_env.crud.crud_test_user import TestUserCRUD +from backend_api.app.tests.test_simulating_env.crud.crud_test_user import TestUserCRUD class TestUserCRUD(TestUserCRUD): diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py index dc4d1231..1c8cb1c9 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_base.py @@ -1,18 +1,17 @@ from collections.abc import Callable import pytest -from sqlalchemy.orm import Session - -from app.crud.base import ( +from backend_api.app.crud.base import ( CRUDBase, ModelType, ) -from app.exceptions.model_exceptions.db_exception import ( +from backend_api.app.exceptions.model_exceptions.db_exception import ( DbObjectAlreadyExistsError, DbObjectDoesNotExistError, ) -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import get_ignore_keys +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import get_ignore_keys +from sqlalchemy.orm import Session @pytest.mark.usefixtures("clear_db", autouse=True) diff --git a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py index 5421109b..150d5099 100644 --- a/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py +++ b/src/backend_api/app/tests/test_simulating_env/crud/crud_test_user.py @@ -1,12 +1,11 @@ import pytest +from backend_api.app.core.schemas import UserCreate, UserUpdate +from backend_api.app.core.security import verify_password +from backend_api.app.crud import CRUD_user as crud +from backend_api.app.tests.test_simulating_env.base_test import BaseTest +from backend_api.app.tests.utils.utils import random_email, random_lower_string from sqlalchemy.orm import Session -from app.core.schemas import UserCreate, UserUpdate -from app.core.security import verify_password -from app.crud import CRUD_user as crud -from app.tests.test_simulating_env.base_test import BaseTest -from app.tests.utils.utils import random_email, random_lower_string - @pytest.mark.usefixtures("clear_db", autouse=True) class TestUserCRUD(BaseTest): diff --git a/src/backend_api/app/tests/test_simulating_env/setup_test_database.py b/src/backend_api/app/tests/test_simulating_env/setup_test_database.py index 979dffd6..822852ec 100644 --- a/src/backend_api/app/tests/test_simulating_env/setup_test_database.py +++ b/src/backend_api/app/tests/test_simulating_env/setup_test_database.py @@ -1,7 +1,6 @@ +from backend_api.app.core.config import settings from pydantic import PostgresDsn -from app.core.config import settings - TEST_DATABASE_URL: PostgresDsn | None = str(settings.TEST_DATABASE_URI) ASYNC_TEST_DATABASE_URL: PostgresDsn | None = str(settings.ASYNC_TEST_DATABASE_URI) diff --git a/src/backend_api/app/tests/utils/database_utils.py b/src/backend_api/app/tests/utils/database_utils.py index e41185f1..58b4e61d 100644 --- a/src/backend_api/app/tests/utils/database_utils.py +++ b/src/backend_api/app/tests/utils/database_utils.py @@ -1,10 +1,9 @@ +from backend_api.app.core.config import settings +from backend_api.app.core.models.database import engine as src_db_engine +from backend_api.app.core.models.models import User from sqlalchemy import Engine, MetaData, delete from sqlalchemy.exc import SQLAlchemyError -from app.core.config import settings -from app.core.models.database import engine as src_db_engine -from app.core.models.models import User - src_db_metadata = MetaData() test_db_metadata = MetaData() diff --git a/src/backend_api/app/tests/utils/model_utils/__init__.py b/src/backend_api/app/tests/utils/model_utils/__init__.py index 98c8f19e..17de52a6 100644 --- a/src/backend_api/app/tests/utils/model_utils/__init__.py +++ b/src/backend_api/app/tests/utils/model_utils/__init__.py @@ -1,20 +1,20 @@ -from app.tests.utils.model_utils.currency import ( +from backend_api.app.tests.utils.model_utils.currency import ( create_random_currency_dict, generate_random_currency, ) -from app.tests.utils.model_utils.item_base_type import ( +from backend_api.app.tests.utils.model_utils.item_base_type import ( create_random_item_base_type_dict, generate_random_item_base_type, ) -from app.tests.utils.model_utils.item_modifier import ( +from backend_api.app.tests.utils.model_utils.item_modifier import ( create_random_item_modifier_dict, generate_random_item_modifier, ) -from app.tests.utils.model_utils.item import ( +from backend_api.app.tests.utils.model_utils.item import ( create_random_item_dict, generate_random_item, ) -from app.tests.utils.model_utils.modifier import ( +from backend_api.app.tests.utils.model_utils.modifier import ( create_random_modifier_dict, generate_random_modifier, ) diff --git a/src/backend_api/app/tests/utils/model_utils/currency.py b/src/backend_api/app/tests/utils/model_utils/currency.py index 7b733aba..9095e3c3 100644 --- a/src/backend_api/app/tests/utils/model_utils/currency.py +++ b/src/backend_api/app/tests/utils/model_utils/currency.py @@ -1,14 +1,14 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import Currency, League -from app.core.schemas import CurrencyCreate -from app.tests.utils.model_utils.league import generate_random_league -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import Currency, League +from backend_api.app.core.schemas import CurrencyCreate +from backend_api.app.tests.utils.model_utils.league import generate_random_league +from backend_api.app.tests.utils.utils import ( random_float, random_int, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud async def create_random_currency_dict( diff --git a/src/backend_api/app/tests/utils/model_utils/item.py b/src/backend_api/app/tests/utils/model_utils/item.py index c9dde2dd..7ec0ee66 100644 --- a/src/backend_api/app/tests/utils/model_utils/item.py +++ b/src/backend_api/app/tests/utils/model_utils/item.py @@ -1,18 +1,20 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import Currency, Item, ItemBaseType, League -from app.core.schemas.item import ItemCreate -from app.tests.utils.model_utils.currency import generate_random_currency -from app.tests.utils.model_utils.item_base_type import generate_random_item_base_type -from app.tests.utils.model_utils.league import generate_random_league -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import Currency, Item, ItemBaseType, League +from backend_api.app.core.schemas.item import ItemCreate +from backend_api.app.tests.utils.model_utils.currency import generate_random_currency +from backend_api.app.tests.utils.model_utils.item_base_type import ( + generate_random_item_base_type, +) +from backend_api.app.tests.utils.model_utils.league import generate_random_league +from backend_api.app.tests.utils.utils import ( random_bool, random_float, random_int, random_json, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud async def create_random_item_dict( diff --git a/src/backend_api/app/tests/utils/model_utils/item_base_type.py b/src/backend_api/app/tests/utils/model_utils/item_base_type.py index 05b8c4c6..a4fe06a2 100644 --- a/src/backend_api/app/tests/utils/model_utils/item_base_type.py +++ b/src/backend_api/app/tests/utils/model_utils/item_base_type.py @@ -1,9 +1,9 @@ +from backend_api.app.core.models.models import ItemBaseType as model_ItemBaseType +from backend_api.app.core.schemas.item_base_type import ItemBaseTypeCreate +from backend_api.app.tests.utils.utils import random_int, random_lower_string from sqlalchemy.orm import Session from app import crud -from app.core.models.models import ItemBaseType as model_ItemBaseType -from app.core.schemas.item_base_type import ItemBaseTypeCreate -from app.tests.utils.utils import random_int, random_lower_string def create_random_item_base_type_dict() -> dict: diff --git a/src/backend_api/app/tests/utils/model_utils/item_modifier.py b/src/backend_api/app/tests/utils/model_utils/item_modifier.py index d28735bf..8bc85c78 100644 --- a/src/backend_api/app/tests/utils/model_utils/item_modifier.py +++ b/src/backend_api/app/tests/utils/model_utils/item_modifier.py @@ -1,7 +1,4 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Currency, Item, ItemBaseType, @@ -9,10 +6,13 @@ League, Modifier, ) -from app.core.schemas.item_modifier import ItemModifierCreate -from app.tests.utils.model_utils.item import generate_random_item -from app.tests.utils.model_utils.modifier import generate_random_modifier -from app.tests.utils.utils import random_float, random_int +from backend_api.app.core.schemas.item_modifier import ItemModifierCreate +from backend_api.app.tests.utils.model_utils.item import generate_random_item +from backend_api.app.tests.utils.model_utils.modifier import generate_random_modifier +from backend_api.app.tests.utils.utils import random_float, random_int +from sqlalchemy.orm import Session + +from app import crud async def create_random_item_modifier_dict( diff --git a/src/backend_api/app/tests/utils/model_utils/league.py b/src/backend_api/app/tests/utils/model_utils/league.py index aa2fc308..8e7ebdee 100644 --- a/src/backend_api/app/tests/utils/model_utils/league.py +++ b/src/backend_api/app/tests/utils/model_utils/league.py @@ -1,13 +1,13 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import League -from app.core.schemas import LeagueCreate -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import League +from backend_api.app.core.schemas import LeagueCreate +from backend_api.app.tests.utils.utils import ( random_datetime, random_float, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud def create_random_league_dict() -> dict: diff --git a/src/backend_api/app/tests/utils/model_utils/modifier.py b/src/backend_api/app/tests/utils/model_utils/modifier.py index e78f0941..0573e021 100644 --- a/src/backend_api/app/tests/utils/model_utils/modifier.py +++ b/src/backend_api/app/tests/utils/model_utils/modifier.py @@ -1,14 +1,14 @@ -from sqlalchemy.orm import Session - -from app import crud -from app.core.models.models import Modifier -from app.core.schemas.modifier import ModifierCreate -from app.tests.utils.utils import ( +from backend_api.app.core.models.models import Modifier +from backend_api.app.core.schemas.modifier import ModifierCreate +from backend_api.app.tests.utils.utils import ( random_bool, random_float, random_int, random_lower_string, ) +from sqlalchemy.orm import Session + +from app import crud def create_random_modifier_dict() -> dict: diff --git a/src/backend_api/app/tests/utils/model_utils/plot.py b/src/backend_api/app/tests/utils/model_utils/plot.py index f71e91e6..0a644851 100644 --- a/src/backend_api/app/tests/utils/model_utils/plot.py +++ b/src/backend_api/app/tests/utils/model_utils/plot.py @@ -1,16 +1,15 @@ from typing import Any -from sqlalchemy.orm import Session - -from app.core.models.models import ( +from backend_api.app.core.models.models import ( League as model_League, ) -from app.core.models.models import ( +from backend_api.app.core.models.models import ( Modifier as model_Modifier, ) -from app.tests.utils.model_utils.item_modifier import ( +from backend_api.app.tests.utils.model_utils.item_modifier import ( generate_random_item_modifier, ) +from sqlalchemy.orm import Session async def create_minimal_random_plot_query_dict(db: Session) -> dict[str, Any]: diff --git a/src/backend_api/app/tests/utils/user.py b/src/backend_api/app/tests/utils/user.py index 0de73fb9..dd2bbbec 100644 --- a/src/backend_api/app/tests/utils/user.py +++ b/src/backend_api/app/tests/utils/user.py @@ -1,12 +1,11 @@ +from backend_api.app.core.config import settings +from backend_api.app.core.schemas import User, UserCreate, UserUpdate +from backend_api.app.crud import CRUD_user +from backend_api.app.tests.utils.utils import random_email, random_lower_string from httpx import AsyncClient from pydantic import EmailStr from sqlalchemy.orm import Session -from app.core.config import settings -from app.core.schemas import User, UserCreate, UserUpdate -from app.crud import CRUD_user -from app.tests.utils.utils import random_email, random_lower_string - async def user_authentication_headers( *, async_client: AsyncClient, email: str, password: str diff --git a/src/backend_api/app/tests/utils/utils.py b/src/backend_api/app/tests/utils/utils.py index 405a6ed6..f65d5d27 100644 --- a/src/backend_api/app/tests/utils/utils.py +++ b/src/backend_api/app/tests/utils/utils.py @@ -5,12 +5,11 @@ from inspect import iscoroutinefunction from typing import Any +from backend_api.app.core.config import settings +from backend_api.app.crud.base import ModelType from httpx import AsyncClient from sqlalchemy import inspect -from app.core.config import settings -from app.crud.base import ModelType - def random_lower_string(*, small_string: bool | None = None) -> str: """Generate a random lowercase string. diff --git a/src/backend_api/app/utils/timing_tracker.py b/src/backend_api/app/utils/timing_tracker.py index fac64659..a1319163 100644 --- a/src/backend_api/app/utils/timing_tracker.py +++ b/src/backend_api/app/utils/timing_tracker.py @@ -3,7 +3,7 @@ from time import perf_counter from typing import ParamSpec, TypeVar -from app.logs.logger import logger +from backend_api.app.logs.logger import logger Param = ParamSpec("Param") RetType = TypeVar("RetType") diff --git a/src/backend_api/app/utils/user.py b/src/backend_api/app/utils/user.py index ce66e523..a74f11ed 100644 --- a/src/backend_api/app/utils/user.py +++ b/src/backend_api/app/utils/user.py @@ -3,12 +3,11 @@ from typing import Any import emails # type: ignore +from backend_api.app.core.config import settings +from backend_api.app.logs.logger import logger from jinja2 import Template from pydantic import EmailStr -from app.core.config import settings -from app.logs.logger import logger - @dataclass class EmailData: diff --git a/src/backend_api/app/validation/turnstile_validation.py b/src/backend_api/app/validation/turnstile_validation.py index f79ab0e1..df86f263 100644 --- a/src/backend_api/app/validation/turnstile_validation.py +++ b/src/backend_api/app/validation/turnstile_validation.py @@ -1,11 +1,12 @@ import httpx # Use httpx instead of requests +from backend_api.app.api.api_message_util import ( + get_failed_send_challenge_request_error_msg, +) +from backend_api.app.core.config import settings +from backend_api.app.core.schemas import TurnstileQuery, TurnstileResponse from fastapi import HTTPException from pydantic import TypeAdapter -from app.api.api_message_util import get_failed_send_challenge_request_error_msg -from app.core.config import settings -from app.core.schemas import TurnstileQuery, TurnstileResponse - class ValidateTurnstileRequest: def __init__(self): diff --git a/src/backend_api/scripts/prestart.sh b/src/backend_api/scripts/prestart.sh index 03cd269b..00a49cf8 100644 --- a/src/backend_api/scripts/prestart.sh +++ b/src/backend_api/scripts/prestart.sh @@ -4,12 +4,12 @@ set -e set -x # Let the DB start -python /app/app/backend_pre_start.py +python /app/backend_api/app/backend_pre_start.py # Run migrations alembic upgrade head # Create initial data in DB -python /app/app/initial_data.py +python /app/backend_api/app/initial_data.py -fastapi run --reload "app/main.py" +fastapi run --reload "backend_api/app/main.py" diff --git a/src/backend_data_retrieval/Dockerfile b/src/backend_data_retrieval/Dockerfile index 5c271593..f2cc6b43 100644 --- a/src/backend_data_retrieval/Dockerfile +++ b/src/backend_data_retrieval/Dockerfile @@ -30,6 +30,11 @@ ENV PYTHONPATH=/data_retrieval_app COPY ./prestart.sh ./pyproject.toml ./uv.lock /data_retrieval_app/ +# For referencing API response schemas +COPY --from=backend_api \ + ./app/core/schemas/ \ + /data_retrieval_app/backend_api/app/core/schemas/ + ADD ./data_retrieval_app /data_retrieval_app/data_retrieval_app # Sync the project diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py new file mode 100644 index 00000000..98dc13b4 --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/currency/currency_type_data_depositor.py @@ -0,0 +1,176 @@ +from collections.abc import Iterator + +from backend_api.app.core.schemas.currency import ( + CurrencyType, + CurrencyTypeCreate, + CurrencyTypeUpdate, +) +from pydantic import TypeAdapter + +from data_retrieval_app.data_deposit.data_depositor_base import DataDepositorBase +from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.currency import ( + ExchangeRatioItem, +) +from data_retrieval_app.logs.logger import data_deposit_logger as logger +from data_retrieval_app.pom_api_authentication import get_superuser_token_headers +from data_retrieval_app.utils import get_data_safe, send_request_safe + + +class CurrencyTypeDataDepositor(DataDepositorBase): + def __init__(self): + self.base_url = settings.BACKEND_BASE_URL + self.pom_auth_headers = get_superuser_token_headers(self.base_url) + + self.data_url = f"{self.base_url}/currency/type/" + + self.name_to_trade_name = self._get_name_to_trade_name_dict() + + def _get_name_to_trade_name_dict(self) -> dict[str, str]: + """ + Retrieves a map for "fancy" currency names, as used in the API, to their trade names, which we need. + """ + headers = { + "User-Agent": f"OAuth pathofmodifiers/{settings.TAG} (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" + } + response = get_data_safe( + "https://www.pathofexile.com/api/trade/data/static", + headers=headers, + logger=logger, + ) + + response_json = response.json() + result = response_json["result"] + currencies = dict[str, str]() + for category in result: + if category["id"] == "Currency": + for entry in category["entries"]: + name = entry["text"] + trade_name = entry["id"] + currencies[name] = trade_name + + return currencies + + def _get_current_currency_types(self) -> list[CurrencyType]: + response = get_data_safe( + self.data_url, + headers=self.pom_auth_headers, + logger=logger, + ) + + return TypeAdapter(list[CurrencyType]).validate_python(response.json()) + + def _load_data(self) -> Iterator[list[ExchangeRatioItem]]: + response = get_data_safe( + "https://api.poe.watch/exchange/ratios?league=Standard&game=poe1", + logger=logger, + ) + response_json = response.json() + currency_items = list[ExchangeRatioItem]() + currency_item_adaptor = TypeAdapter(ExchangeRatioItem) + for item in response_json["items"]: + if item["category"] == "currency": + item["leagueId"] = -1 + currency_items.append(currency_item_adaptor.validate_python(item)) + + yield currency_items + + def _process_data( + self, currency_items: list[ExchangeRatioItem] + ) -> list[CurrencyTypeCreate]: + """ + Scenarios that requires updated currency: + Name is: + - 'TEMP_NAME': this is right after the 'ee1239y6yfda' migration, + as the database has no knowledge of name + - not the same when trade name is + Trade name is: + - not the same when name is + Incoming currency always takes precedence + """ + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_auth_headers) + + current_currencies = self._get_current_currency_types() + processed_currencies = list[CurrencyTypeCreate]() + + found_chaos = False + for currency in currency_items: + trade_name = self.name_to_trade_name.get(currency.name) + if trade_name is None: + continue + + currency_id = -1 + + found_duplicate = False + need_update = False + for current_currency in current_currencies: + if ( + current_currency.name == "TEMP_NAME" + and current_currency.tradeName == trade_name + ): + need_update = True + if ( + current_currency.name == currency.name + and current_currency.tradeName != trade_name + ): + need_update = True + if ( + current_currency.name != currency.name + and current_currency.tradeName == trade_name + ): + need_update = True + if ( + current_currency.name == currency.name + and current_currency.tradeName == trade_name + ): + found_duplicate = True + + if current_currency.tradeName == "chaos": + found_chaos = True + + if found_duplicate or need_update: + currency_id = current_currency.currencyId + break + + if need_update: + updated_currency = CurrencyTypeUpdate( + currencyId=currency_id, name=currency.name, tradeName=trade_name + ) + + send_request_safe( + "put", + self.data_url, + json=updated_currency.model_dump(), + headers=self.pom_auth_headers, + logger=logger, + ) + + elif not found_duplicate: + processed_currencies.append( + CurrencyTypeCreate(name=currency.name, tradeName=trade_name) + ) + + if not found_chaos: + processed_currencies.append( + CurrencyTypeCreate(name="Chaos Orb", tradeName="chaos") + ) + + return processed_currencies + + def _insert_data(self, currencies: list[CurrencyTypeCreate]): + if not currencies: + return + + logger.info("Inserting data into database.") + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_auth_headers) + + send_request_safe( + "post", + self.data_url, + json=TypeAdapter(list[CurrencyTypeCreate]).dump_python(currencies), + headers=headers, + ) + + logger.info("Successfully inserted data into database.") diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py index b40c95b1..be85da2f 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/data_depositor_base.py @@ -14,7 +14,10 @@ class DataDepositorBase: - def __init__(self, data_type: Literal["modifier", "item_base_type"]) -> None: + def __init__( + self, + data_type: Literal["modifier", "item_base_type", "league"], + ) -> None: self.base_url = settings.BACKEND_BASE_URL self.pom_auth_headers = get_superuser_token_headers(self.base_url) diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py index 342c0bbc..0dc20f6c 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/main.py @@ -1,3 +1,6 @@ +from data_retrieval_app.data_deposit.currency.currency_type_data_depositor import ( + CurrencyTypeDataDepositor, +) from data_retrieval_app.data_deposit.data_depositor_base import DataDepositorBase from data_retrieval_app.data_deposit.item_base_type.item_base_type_data_depositor import ( ItemBaseTypeDataDepositor, @@ -19,6 +22,7 @@ def main(): "modifer": ModifierDataDepositor(), "itemBaseType": ItemBaseTypeDataDepositor(), "league": LeagueDataDepositor(), + "currencyType": CurrencyTypeDataDepositor(), } for key, data_depositor in data_depositors.items(): logger.info(f"Depositing {key} data.") diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py index d9c606af..94fc60ef 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_data_depositor.py @@ -1,200 +1,221 @@ -from io import StringIO - -import pandas as pd -import requests +import csv +import os +from collections.abc import Iterator + +from backend_api.app.core.schemas.modifier import ( + GroupedModifier, + ModifierCreate, + ModifierRoll, + ModifierUpdate, +) +from pydantic import TypeAdapter from data_retrieval_app.data_deposit.data_depositor_base import DataDepositorBase from data_retrieval_app.data_deposit.modifier.modifier_processing_modules import ( ModifierRegexCreator, - check_for_additional_modifier_types, - check_for_new_related_unique, - check_for_updated_numerical_rolls, - check_for_updated_text_rolls, - do_update_regex, ) from data_retrieval_app.logs.logger import data_deposit_logger as logger -from data_retrieval_app.utils import df_to_JSON, get_data_safe - -CASCADING_UPDATE = True +from data_retrieval_app.utils import send_request_safe class ModifierDataDepositor(DataDepositorBase): def __init__(self) -> None: super().__init__(data_type="modifier") - - self.modifier_types = [ - "implicit", - "explicit", - "delve", - "fractured", - "synthesised", - "unique", - "corrupted", - "enchanted", - "veiled", - ] self.regex_creator = ModifierRegexCreator() - self.update_disabled = not CASCADING_UPDATE - - def _get_current_modifiers(self) -> pd.DataFrame: - logger.info("Retrieving previously deposited data.") - - response = get_data_safe( - self.data_url, headers=self.pom_auth_headers, logger=logger + def _get_current_modifiers(self) -> dict[str, GroupedModifier]: + response = send_request_safe( + "get", + f"{self.data_url}grouped/", + headers=self.pom_auth_headers, + logger=logger, ) - json_io = StringIO(response.content.decode("utf-8")) - df = pd.read_json(json_io, dtype=str) - - if df.empty: - logger.info("Found no previously deposited data.") - return None - else: - logger.info("Successfully retrieved previously deposited data.") - return df - - def _update_duplicates( - self, duplicate_df: pd.DataFrame, current_modifiers_df: pd.DataFrame - ) -> None: - if self.update_disabled: - return None - logger.info("Checking if duplicates contain updated information.") - - current_duplicate_modifiers_df = current_modifiers_df.loc[ - current_modifiers_df["effect"].isin(duplicate_df["effect"]) - ].copy() - - # We sort them so that they line up. - # We go through in reverse, as we wish to start with the row that has the highest position. - current_duplicate_modifiers_df.sort_values( - by=["effect", "position"], ascending=False, inplace=True - ) - duplicate_df.sort_values( - by=["effect", "position"], ascending=False, inplace=True - ) - - update_url = self.data_url + "?modifierId={}&position={}" + modifiers = TypeAdapter(list[GroupedModifier]).validate_python(response.json()) + + return {modifier.effect: modifier for modifier in modifiers} + + def _check_for_updates( + self, modifier: ModifierCreate, current_modifier: GroupedModifier + ): + need_update = False + updated_modifier = ModifierUpdate(modifierId=current_modifier.modifierId) + for field in modifier.model_fields: + if field == "rolls": + continue + if field == "relatedUniques": + new = modifier.relatedUniques.split("|") + old = current_modifier.relatedUniques.split("|") + related_uniques = set(new).difference(set(old)) + if related_uniques: + updated_modifier.relatedUniques = "|".join(set(new) | set(old)) + need_update = True + + continue + + if field == "regex": + continue + + new = getattr(modifier, field) + old = getattr(current_modifier, field) + if not (new == old or (isinstance(old, bool) and old)): + # update rows which have a different value + # ignore rows which are bools that are already True + setattr(updated_modifier, field, new) + need_update = True - rolls = None update_regex = False - for (_, row_cur), (_, row_new) in zip( - current_duplicate_modifiers_df.iterrows(), - duplicate_df.iterrows(), - strict=False, + for new_roll, old_roll in zip( + modifier.rolls, current_modifier.rolls, strict=True ): - put_update = False - data = df_to_JSON(row_cur, request_method="put") - position = int(data["position"]) - # if position is higher than 1, we want to store the types of rolls it has - if position >= 1 and rolls is None: - update_regex = False - effect = data["effect"] - same_modifier_df = duplicate_df.loc[ - duplicate_df["effect"] == effect - ].copy() - same_modifier_df.sort_values( - by="position", inplace=True - ) # So that the rolls are added in the correct order - rolls = [] - for _, same_modifier_row in same_modifier_df.iterrows(): - if not pd.isna(same_modifier_row["static"]): - pass - elif not pd.isna(same_modifier_row["textRolls"]): - rolls.append(same_modifier_row["textRolls"]) - else: - rolls.append(None) - - if "updatedAt" in data: - data.pop("updatedAt") - - if not pd.isna(row_new["static"]): - pass - elif not pd.isna(row_new["textRolls"]): - data, put_update, rolls = check_for_updated_text_rolls( - data=data, - row_new=row_new, - rolls=rolls, - regex_creator=self.regex_creator, + updated_roll = ModifierRoll(position=new_roll.position) + roll_need_update = False + if new_roll.minRoll is not None and new_roll.minRoll < old_roll.minRoll: + updated_roll.minRoll = new_roll.minRoll + roll_need_update = True + + if new_roll.maxRoll is not None and new_roll.maxRoll > old_roll.maxRoll: + updated_roll.maxRoll = new_roll.maxRoll + roll_need_update = True + + if new_roll.textRolls is not None: + text_rolls = set(new_roll.textRolls).difference(set(old_roll.textRolls)) + if text_rolls: + # preserve order to not disturb existing data + updated_roll.textRolls = old_roll.textRolls + list(text_rolls) + roll_need_update = True + update_regex = True # regex contains text rolls + + if roll_need_update: + if updated_modifier.rolls is None: + updated_modifier.rolls = list[ModifierRoll]() + + updated_modifier.rolls.append(updated_roll) + need_update = True + + if update_regex: + updated_modifier.regex = self.regex_creator.make_regex( + modifier.effect, updated_modifier.rolls + ) + need_update = True + + if need_update: + send_request_safe( + "put", + self.data_url, + json=updated_modifier.model_dump(exclude_none=True, exclude_unset=True), + headers=self.pom_auth_headers, + logger=logger, + ) + + return need_update + + def _remove_duplicates( + self, modifiers: list[ModifierCreate] + ) -> list[ModifierCreate]: + current_modifiers = self._get_current_modifiers() + + previous_effects = list[str]() + did_update = False + for modifier in modifiers[:]: + if modifier.effect in current_modifiers: + did_update = self._check_for_updates( + modifier, current_modifiers[modifier.effect] ) - update_regex = put_update - else: - data, put_update = check_for_updated_numerical_rolls( - data=data, row_new=row_new + modifiers.remove(modifier) + continue + + if modifier.effect in previous_effects: + modifiers.remove(modifier) + continue + + previous_effects.append(modifier.effect) + if did_update: + logger.info("Updated modifiers using new data") + return modifiers + + def _track_comments(self, modifiers: list[ModifierCreate]) -> list[ModifierCreate]: + unique_name = self.logged_file_comments["Unique Name"] + for modifier in modifiers: + modifier.relatedUniques = unique_name + + return modifiers + + def _load_data(self) -> Iterator[ModifierCreate]: + for filename in os.listdir(self.new_data_location): + modifiers = list[dict]() + filepath = os.path.join(self.new_data_location, filename) + + self.logged_file_comments = {} + logger.info(f"Loading new data from '{filename}'.") + with open(filepath) as infile: + while True: + position = infile.tell() + line = infile.readline() + + if not line: + break + + if line.startswith("#"): + logger.info(line.rstrip()) + split_line = line[1:].split(":", 1) + self.logged_file_comments[split_line[0].strip()] = split_line[ + 1 + ].strip() + else: + # We found the CSV header, so go back to its beginning + infile.seek(position) + break + + modifiers.extend(csv.DictReader(infile)) + + deposit_modifiers = dict[str, ModifierCreate]() + for modifier in modifiers: + for key in list(modifier.keys()): + if modifier[key] == "": + modifier.pop(key) + + text_roll: str | None = modifier.get("textRolls") + if text_roll is not None: + text_roll = text_roll.split("|") + + roll = ModifierRoll( + position=modifier["position"], + minRoll=modifier.get("minRoll"), + maxRoll=modifier.get("maxRoll"), + textRolls=text_roll, ) - if update_regex: - data = do_update_regex(data, rolls, regex_creator=self.regex_creator) - put_update = True + deposit_modifier = deposit_modifiers.get(modifier["effect"]) + if deposit_modifier is None: + deposit_modifier = ModifierCreate(**modifier) + deposit_modifiers[modifier["effect"]] = deposit_modifier - data, put_update = check_for_additional_modifier_types( - data=data, - put_update=put_update, - row_new=row_new, - modifier_types=self.modifier_types, - ) + deposit_modifier.rolls.append(roll) - data, put_update = check_for_new_related_unique( - data=data, - put_update=put_update, - new_related_unique=self.logged_file_comments["Unique Name"], + yield TypeAdapter(list[ModifierCreate]).validate_python( + deposit_modifiers.values() ) - if put_update: - logger.info("Pushed updated modifier to the database.") - headers = { - "accept": "application/json", - "Content-Type": "application/json", - } - headers.update(self.pom_auth_headers) - try: - response = requests.put( - update_url.format(row_cur["modifierId"], row_cur["position"]), - json=data, - headers=headers, - # add HTTP Basic Auth - ) - response.raise_for_status() - except Exception as e: - logger.error( - f"The following error occurred while making request during _update_duplicates modifiers: {e}" - ) - raise e - - # We reset the rolls if the position is 0, because then the next row will be a new modifier - if position == 0 and rolls is not None: - rolls = None - update_regex = False - - def _remove_duplicates(self, new_modifiers_df: pd.DataFrame) -> pd.DataFrame: - current_modifiers_df = self._get_current_modifiers() - - new_modifiers_df = new_modifiers_df.drop_duplicates() - - if current_modifiers_df is None: - logger.info("Skipping duplicate removing due to no previous data") - return new_modifiers_df - - logger.info("Removing duplicate modifiers") - duplicate_mask = ( - new_modifiers_df["effect"] - .str.lower() - .isin(current_modifiers_df["effect"].str.lower()) + def _process_data(self, modifiers: list[ModifierCreate]) -> list[ModifierCreate]: + modifiers = self.regex_creator.add_regex(modifiers) + modifiers = self._track_comments(modifiers) + modifiers = self._remove_duplicates(modifiers) + return modifiers + + def _insert_data(self, modifiers: list[ModifierCreate]): + if not modifiers: + return + + logger.info("Inserting data into database.") + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_auth_headers) + + send_request_safe( + "post", + self.data_url, + json=TypeAdapter(list[ModifierCreate]).dump_python(modifiers), + headers=headers, ) - duplicate_df = new_modifiers_df.loc[duplicate_mask].copy() - self._update_duplicates(duplicate_df, current_modifiers_df) - non_duplicate_df = new_modifiers_df.loc[~duplicate_mask].copy() - - return non_duplicate_df - - def _track_comments(self, df: pd.DataFrame) -> pd.DataFrame: - df["relatedUniques"] = self.logged_file_comments["Unique Name"] - - return df - - def _process_data(self, df: pd.DataFrame) -> pd.DataFrame: - df = self.regex_creator.add_regex(df.copy()) - df = self._remove_duplicates(df.copy()) - df = self._track_comments(df.copy()) - return df + logger.info("Successfully inserted data into database.") diff --git a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py index 164b80af..631dd1a2 100644 --- a/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py +++ b/src/backend_data_retrieval/data_retrieval_app/data_deposit/modifier/modifier_processing_modules.py @@ -1,342 +1,24 @@ import re -from typing import Any -import pandas as pd - -from data_retrieval_app.logs.logger import data_deposit_logger as logger +from backend_api.app.core.schemas.modifier import ModifierCreate, ModifierRoll class ModifierRegexCreator: - def __init__(self): - self.modifier_df_required_columns = [ - "minRoll", - "maxRoll", - "textRolls", - "position", - "effect", - "static", - ] - - def _divide_into_dynamic_static( - self, modifier_df: pd.DataFrame - ) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - Divides the modifier dataframe based on the `static` attribute. - """ - logger.debug("Dividing modifier dataframe into dynamic and static modifiers") - - static_modifier_mask = modifier_df["static"] == "True" - - dynamic_modifier_df = ( - modifier_df.loc[~static_modifier_mask].copy().reset_index() - ) - static_modifier_df = modifier_df.loc[static_modifier_mask].copy().reset_index() - - logger.debug("Successfully divided modifier dataframe") - - return dynamic_modifier_df, static_modifier_df - - def _prepare_df(self, dynamic_modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Escapes the `+`, as it is a regex quantifier. - """ - dynamic_modifier_df["effect"] = dynamic_modifier_df["effect"].str.replace( - "+", r"\+" - ) # Do we need to do the same for `-`? Can other regex symbols occur? - - return dynamic_modifier_df - - def _pre_process( - self, modifier_df: pd.DataFrame - ) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - Performs pre-process steps: - 1. The program assumes certain columns are present to work. These - columns are filled with `pd.NA` if not present. - 2a. Divides the modifiers based on their `static` attribute. - 2b. Returns only the static modifier df if no dyamic modifiers - are found. - 3. Prepares the dynamic modifier df for regex steps. - """ - modifier_df = modifier_df.reindex( - columns=modifier_df.columns.union(self.modifier_df_required_columns) - ) - - dynamic_modifier_df, static_modifier_df = self._divide_into_dynamic_static( - modifier_df=modifier_df - ) - if dynamic_modifier_df.empty: - logger.debug("Only static modifiers present") - return None, static_modifier_df - - dynamic_modifier_df = self._prepare_df(dynamic_modifier_df.copy()) - logger.debug("Finished pre-processing steps") - - return dynamic_modifier_df, static_modifier_df - - def _group_df(self, dynamic_modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Groups the dynamic modifier based on effect. - `position` and `textRolls` are combined into lists of - equal length. - """ - logger.debug( - "Preparing dynamic modifier dataframe for regex conversion, by grouping by effect" - ) - agg_dict = { - "position": lambda positions: list(set(positions)), - "textRolls": lambda rolls: [ - roll if not pd.isna(roll) else None for roll in rolls - ], - } - grouped_dynamic_modifier_df = dynamic_modifier_df.groupby( - "effect", as_index=False, sort=False - ).agg(agg_dict) - logger.debug("Successfully grouped modifiers by effect") - - return grouped_dynamic_modifier_df - - def create_regex_from_row(self, row: pd.DataFrame | dict) -> pd.DataFrame: - """ - A method available for both internal and external use. - - Uses the aggregated `textRolls` field to determine wether - the corresponding position has a numerical (textRoll is None), - or a text roll which determines the regex pattern to insert in - place of the `#`. - - Then it accounts for alternative spelling (eg. `-` instead of `+` or - `reduced` instead of `increased`) - """ - effect: str = row["effect"] - text_rolls: list[str] = row["textRolls"] - - for text_roll in text_rolls: - if text_roll is not None: - effect = effect.replace("#", f"({text_roll})", 1) + def make_regex(self, effect: str, rolls: list[ModifierRoll]) -> str: + regex = effect.replace("+", "[+-]") + for roll in rolls: + if roll.textRolls is not None: + regex = regex.replace("#", f"({"|".join(roll.textRolls)})", 1) else: - effect = effect.replace("#", r"([0-9]*[.]?[0-9]+)", 1) + regex = regex.replace("#", r"([0-9]*[.]?[0-9]+)", 1) - regex = effect.replace(r"\+", r"[+-]") regex = re.sub(r"increased|reduced", "(increased|reduced)", regex) return rf"^{regex}$" - def _unnest_df( - self, - dynamic_modifier_df: pd.DataFrame, - grouped_dynamic_modifier_df: pd.DataFrame, - ) -> pd.DataFrame: - """ - Unnests the grouped modfifier dataframe to original shape. - """ - logger.debug("Reverting the grouped dynamic modifier group to original shape") - exploded_grouped_dynamic_modifier_df = grouped_dynamic_modifier_df.explode( - "position", ignore_index=True - ) - - dynamic_modifier_df["regex"] = exploded_grouped_dynamic_modifier_df["regex"] - logger.debug("Reverted to original shape") - - return dynamic_modifier_df - - def _add_regex(self, dynamic_modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Performs the actual steps of adding regex: - 1. Groups the modifiers for easier processing. - 2. Creates the regex by iterating over each row. - 3. Accounts for alternative wording. - 4. Unnests the grouped modfifier dataframe to original shape. - """ - grouped_dynamic_modifier_df = self._group_df(dynamic_modifier_df.copy()) - - logger.debug("Adding regex row by row") - grouped_dynamic_modifier_df["regex"] = grouped_dynamic_modifier_df.apply( - self.create_regex_from_row, axis=1 - ) - - dynamic_modifier_df = self._unnest_df( - dynamic_modifier_df.copy(), grouped_dynamic_modifier_df.copy() - ) - failed_df = dynamic_modifier_df.loc[dynamic_modifier_df["regex"].isna()] - if not failed_df.empty: - logger.critical("Some modifiers did not get a regex:") - logger.critical(failed_df["effect"]) - raise AssertionError("Some modifiers did not get a regex.") - logger.debug("Successfully added regex row by row") - - return dynamic_modifier_df - - def _post_process( - self, dynamic_modifier_df: pd.DataFrame, static_modifier_df: pd.DataFrame - ) -> pd.DataFrame: - """ - Undos temporary changes. - - Combines dynamic and static modifiers to one modifier dataframe again. - """ - dynamic_modifier_df.loc[:, "effect"] = dynamic_modifier_df[ - "effect" - ].str.replace(r"\+", "+") - - def remove_quantifier(ser: pd.Series) -> pd.Series: - contains_q_mask = ser.str.contains("?", regex=False) - ser_contains_q = ser.loc[contains_q_mask].copy() - - if ser_contains_q.empty: - return ser - - ser_split = ser_contains_q.str.split("?") - for index, str_parts in ser_split.items(): - ser.iloc[index] = "".join( - [ - part[:-1] if i < (len(str_parts) - 1) else part - for i, part in enumerate(str_parts) - ] - ) - return ser - - dynamic_modifier_df.loc[:, "effect"] = remove_quantifier( - dynamic_modifier_df.loc[:, "effect"] - ) - - final_df = pd.concat( - (dynamic_modifier_df, static_modifier_df), ignore_index=True - ) - - return final_df - - def add_regex(self, modifier_df: pd.DataFrame) -> pd.DataFrame: - """ - Processes the incoming dataframe, making it ready for adding regex. - If no dynamic modifiers are present, it returns early. - Otherwise, regex is added and post processing steps are applied - to convert the dataframe back to original shape. - """ - dynamic_modifier_df, static_modifier_df = self._pre_process(modifier_df.copy()) - if dynamic_modifier_df is None: - return static_modifier_df - - dynamic_modifier_df = self._add_regex(dynamic_modifier_df.copy()) - - logger.debug("Finishing up the regex addition step") - final_df = self._post_process( - dynamic_modifier_df.copy(), static_modifier_df.copy() - ) - logger.debug("Finished adding regex.") - - return final_df - - -def check_for_updated_text_rolls( - data: dict[str, Any], - row_new: pd.DataFrame, - rolls: list[None | str], - regex_creator: ModifierRegexCreator, -) -> tuple[dict[str, Any], bool, list[None | str]]: - if data["textRolls"] != row_new["textRolls"]: - logger.info( - f"Found a modifier with new 'textRolls'. Modifier: {data['effect']}" - ) - new_rolls: list[str] = row_new["textRolls"].split("|") - old_rolls: list[str] = data["textRolls"].split("|") - combined_rolls = old_rolls + [ - roll - for roll in new_rolls - if roll.lower() not in [old_roll.lower() for old_roll in old_rolls] - ] - if rolls is not None: - rolls[int(data["position"])] = "|".join(combined_rolls) - data["textRolls"] = rolls - else: - data["textRolls"] = [data["textRolls"]] - - data["effect"] = data["effect"].replace("+", r"\+") - data["regex"] = regex_creator.create_regex_from_row(data) - data["effect"] = data["effect"].replace(r"\+", "+") - - data["textRolls"] = "|".join(combined_rolls) - - put_update = True - else: - put_update = False - - return data, put_update, rolls - - -def do_update_regex( - data: dict[str, Any], rolls: list[None, str], regex_creator: ModifierRegexCreator -) -> dict[str, Any]: - if "textRolls" in data: - text_rolls = data["textRolls"] - else: - text_rolls = None - - data["textRolls"] = rolls - data["effect"] = data["effect"].replace("+", r"\+") - data["regex"] = regex_creator.create_regex_from_row(data) - data["effect"] = data["effect"].replace(r"\+", "+") - - if text_rolls is not None: - data["textRolls"] = text_rolls - else: - data.pop("textRolls") - - return data - - -def check_for_updated_numerical_rolls( - data: dict[str, Any], row_new: pd.DataFrame -) -> tuple[dict[str, Any], bool]: - min_roll = data["minRoll"] - max_roll = data["maxRoll"] - - new_min_roll = row_new["minRoll"] - new_max_roll = row_new["maxRoll"] - - if float(min_roll) > float(new_min_roll): - logger.debug( - f"Found a modifier with a lower 'minRoll'. Modifier: {data['effect']}" - ) - data["minRoll"] = new_min_roll - - if float(max_roll) < float(new_max_roll): - logger.info( - f"Found a modifier with a higher 'maxRoll'. Modifier: {data['effect']}" - ) - data["maxRoll"] = new_max_roll - - if min_roll != new_min_roll or max_roll != new_max_roll: - logger.debug("Updating modifier to bring numerical roll range up-to-date.") - put_update = True - else: - put_update = False - - return data, put_update - - -def check_for_additional_modifier_types( - data: dict[str, Any], - row_new: pd.Series, - put_update: bool, - modifier_types: list[str], -) -> tuple[dict[str, Any], bool]: - for modifier_type in modifier_types: - if modifier_type in row_new.index and modifier_type not in data: - logger.info( - f"Added a modifier type to a modifier. Modifier: {data['effect']}" - ) - data[modifier_type] = row_new[modifier_type] - put_update = True - - return data, put_update - - -def check_for_new_related_unique( - data: dict[str, Any], put_update: bool, new_related_unique: str -) -> tuple[dict[str, Any], bool]: - if new_related_unique not in data["relatedUniques"]: - data["relatedUniques"] += "|" + new_related_unique - put_update = True + def add_regex(self, modifiers: list[ModifierCreate]) -> list[ModifierCreate]: + for modifier in modifiers: + if not modifier.static: + modifier.regex = self.make_regex(modifier.effect, modifier.rolls) - return data, put_update + return modifiers diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py index 1ebbb923..97a98caa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/cache.py @@ -7,7 +7,9 @@ @contextmanager def get_cache(): - cache = redis.from_url(str(settings.CACHE_URI), decode_responses=True) + cache = redis.from_url( + str(settings.DATA_RETRIEVAL_CACHE_URI), decode_responses=True + ) try: yield cache diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py index 10bf8d42..707a0cd0 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py @@ -12,6 +12,8 @@ class Settings(BaseSettings): env_file=".env", env_ignore_empty=True, extra="ignore" ) + TAG: str + DOMAIN: str @computed_field # type: ignore[prop-decorator] @@ -24,18 +26,18 @@ def BACKEND_BASE_URL(self) -> HttpUrl: REDIS_PORT: int = 6379 REDIS_SERVER: str - REDIS_CACHE: str = str(0) + REDIS_BACKEND_DATA_RETRIEVAL: int = 1 REDIS_PASSWORD: str = "" @computed_field # type: ignore[prop-decorator] @property - def CACHE_URI(self) -> AnyUrl: + def DATA_RETRIEVAL_CACHE_URI(self) -> AnyUrl: return MultiHostUrl.build( scheme="redis", password=self.REDIS_PASSWORD, host=self.REDIS_SERVER, port=self.REDIS_PORT, - path=self.REDIS_CACHE, + path=f"{self.REDIS_BACKEND_DATA_RETRIEVAL}", ) MANUAL_NEXT_CHANGE_ID: bool @@ -51,6 +53,8 @@ def CACHE_URI(self) -> AnyUrl: MINI_BATCH_SIZE: int = 30 N_CHECKPOINTS_PER_TRANSFORMATION: int = 10 + MAX_MIRROR_PRICE: int = 5 + MAX_TIME_PER_MINI_BATCH: int = 3 * 60 diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py index a5c1b6e4..3cf20e8c 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py @@ -1,44 +1,136 @@ -from typing import Any +from collections.abc import Sequence -import pandas as pd +from backend_api.app.core.schemas.currency import Currency, CurrencyType +from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter +from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.currency import ( + ExchangeRatioItem, +) +from data_retrieval_app.external_data_retrieval.transforming_data.transform_currency_api_data import ( + TransformCurrencyAPIData, +) from data_retrieval_app.logs.logger import external_data_retrieval_logger as logger -from data_retrieval_app.utils import get_data_safe +from data_retrieval_app.pom_api_authentication import ( + get_superuser_token_headers, +) +from data_retrieval_app.utils import send_request_safe class CurrencyAPIHandler: + backend_base_url = settings.BACKEND_BASE_URL + pom_auth_headers = get_superuser_token_headers(backend_base_url) + + currency_url = f"{backend_base_url}/currency/" + def __init__(self, url: str) -> None: self.url = url - def _json_to_df(self, currencies: list) -> pd.DataFrame: - df = pd.json_normalize(currencies) + self.currency_item_adapter = TypeAdapter(ExchangeRatioItem) + + self.currency_types = self._get_currency_types() + self.transformer = TransformCurrencyAPIData(self.currency_types) + + def _get_currency_types(self) -> dict[str, CurrencyType]: + response = send_request_safe( + "get", + self.currency_url + "type/", + headers=self.pom_auth_headers, + logger=logger, + ) + + currency_types = TypeAdapter(list[CurrencyType]).validate_python( + response.json() + ) + return {currency_type.name: currency_type for currency_type in currency_types} - return df + def _get_latest_currencies( + self, leagues: Sequence[League] + ) -> dict[tuple[int, str], Currency]: + response = send_request_safe( + "get", + self.currency_url + "price/latest/", + params={"league_ids": [league.leagueId for league in leagues]}, + headers=self.pom_auth_headers, + logger=logger, + ) + currency_list = TypeAdapter(list[Currency]).validate_python(response.json()) - def make_request(self, leagues: list[dict[str, Any]]) -> pd.DataFrame: + currencies = dict[tuple[int, str], Currency]() + for currency in currency_list: + currencies[(currency.leagueId, currency.tradeName)] = currency + + return currencies + + def _remove_old_currencies( + self, + currencies: dict[tuple[int, str], Currency], + leagues: list[League], + current_hours: dict[int, int], + ) -> list[League]: """ - Makes an initial, synchronous, API call. + Modifies latest currencies inplace """ - df = None + if not currencies: + # No prior data is available + return leagues + + leagues_needs_new_data = set[int]() + for (league_id, _), currency in currencies.items(): + current_hour = current_hours[league_id] + currency_hour = currency.createdHoursSinceLaunch + if currency_hour != current_hour: + leagues_needs_new_data.add(league_id) + + return [ + league for league in leagues if league.leagueId in leagues_needs_new_data + ] + + def _get_new_data(self, leagues: list[League]) -> list[ExchangeRatioItem]: + currency_items = list[ExchangeRatioItem]() for league in leagues: - response = get_data_safe( - self.url.format(league=league["name"].replace(" ", "+")), logger=logger + response = send_request_safe( + "get", + self.url.format(league=league.name.replace(" ", "+")), + logger=logger, ) response_json = response.json() - items_df = pd.json_normalize(response_json["items"]) - currency_df = items_df[items_df["category"] == "currency"] - currency_df["leagueId"] = league["leagueId"] - if df is None: - df = currency_df - else: - df = pd.concat((df, currency_df)) - return df + for item in response_json["items"]: + if item["category"] == "currency": + item["leagueId"] = league.leagueId + currency_items.append( + self.currency_item_adapter.validate_python(item) + ) + + return currency_items + + def get_currency_data( + self, leagues: list[League], current_hours: dict[int, int] + ) -> dict[tuple[int, str], Currency]: + """ + Returns a dict mapping trade name to list of the same currency per league + """ + currencies = self._get_latest_currencies(leagues) + + leagues_needs_new_data = self._remove_old_currencies( + currencies, leagues, current_hours + ) + + if leagues_needs_new_data: + exchange_ratios = self._get_new_data(leagues_needs_new_data) + currencies.update( + self.transformer.transform_and_insert(exchange_ratios, current_hours) + ) + + return currencies def store_data_to_csv(self, path: str) -> None: """ Stores the data in a CSV. Only to be used for testing purposes. """ - currencies_df = self.make_request() + # currencies_df = self.make_request() - currencies_df.to_csv(path + "/currencies.csv", index=False) + # currencies_df.to_csv(path + "/currencies.csv", index=False) + raise NotImplementedError() diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py index 87435c24..93d2d021 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py @@ -5,24 +5,31 @@ from collections.abc import Iterator from concurrent.futures import Future, ThreadPoolExecutor from queue import Empty, Full, Queue -from typing import Any, Literal +from typing import Literal import httpx import pandas as pd import redis +from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + Stash, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.categorized import ( + OrganizedItemsByCategory, +) from data_retrieval_app.external_data_retrieval.data_retrieval.utils import ( ByteResponse, RateLimiterThreadSafe, ) +from data_retrieval_app.external_data_retrieval.detectors.detector_controller import ( + DetectorController, +) from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( - UniqueArmourDetector, + # UnidentifiedUniqueDetector, UniqueDetector, - UniqueJewelDetector, - UniqueJewelleryDetector, - UniqueUnidentifiedDetector, - UniqueWeaponDetector, ) from data_retrieval_app.external_data_retrieval.utils import ( ProgramTooSlowException, @@ -40,7 +47,7 @@ class PoEAPIHandler: """ headers = { - "User-Agent": f"OAuth pathofmodifiers/0.1.0 (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" + "User-Agent": f"OAuth pathofmodifiers/{settings.TAG} (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" } def __init__( @@ -48,8 +55,8 @@ def __init__( url: str, auth_token: str, *, - leagues: list[dict[str, Any]], - item_detectors: list[UniqueDetector] | None = None, + leagues: list[League], + detector_controller: DetectorController | None = None, ) -> None: """ Parameters: @@ -59,15 +66,14 @@ def __init__( """ logger.debug("Initializing PoEAPIHandler.") self.leagues = leagues - if item_detectors is None: - item_detectors = [ - UniqueArmourDetector(leagues), - UniqueJewelDetector(leagues), - UniqueJewelleryDetector(leagues), - UniqueWeaponDetector(leagues), - UniqueUnidentifiedDetector(leagues), - ] - logger.debug("Item detectors set to: " + str(item_detectors)) + if detector_controller is None: + detector_controller = DetectorController( + [ + UniqueDetector(), + # UnidentifiedUniqueDetector(), + ], + leagues, + ) self.url = url logger.debug("Url set to: " + self.url) self.auth_token = auth_token @@ -75,75 +81,13 @@ def __init__( logger.debug("Headers set to: " + str(self.headers)) - self.item_detectors = item_detectors - logger.debug("Item detectors set to: " + str(self.item_detectors)) + self.detector_controller = detector_controller + + self.stashes_adapter = TypeAdapter(list[Stash]) self.skip_program_too_slow = False logger.info("PoEAPIHandler successfully initialized.") - def _json_to_df(self, stashes: list) -> pd.DataFrame | None: - df_temp = pd.json_normalize(stashes) - - if "items" not in df_temp.columns: - return None - - df_temp = df_temp.explode(["items"]) - - df_temp = df_temp.loc[~df_temp["items"].isnull()] - - df_temp.drop("items", axis=1, inplace=True) - - df = pd.json_normalize(stashes, record_path=["items"]) - - df["stash_index"] = df_temp.index - - df_temp.index = df.index - - df[df_temp.columns.to_list()] = df_temp - - return df - - def _detector_filter(self, stashes: list) -> pd.DataFrame: - """ - Parameters: - :param stashes: (list) A list of stash objects - """ - df_wanted = pd.DataFrame() - n_new_items = 0 - n_total_unique_items = 0 - df = self._json_to_df(stashes) - if df is None: - return df_wanted - - # The stashes are fed to all item detectors, slowly being filtered down - try: - for item_detector in self.item_detectors: - ( - df_filtered, - item_count, - n_unique_found_items, - df_leftover, - ) = item_detector.iterate_stashes(df) - - df_wanted = pd.concat((df_wanted, df_filtered)) - - del df_filtered - - n_new_items += item_count - n_total_unique_items += n_unique_found_items - if df_leftover.empty: - break - - df = df_leftover.copy(deep=True) - del df_leftover - except Exception as e: - logger.exception( - f"While checking stashes (detector: {item_detector}), this exception occured: {e}" - ) - raise - - return df_wanted.reset_index() - def _get_latest_change_id(self) -> str: """ Gets the latest change id from GGG. @@ -192,9 +136,7 @@ def _follow_stream( # pick up from latest checkpoint if listener_id == 0: logger.debug("Main listener initiating the ping-pong again") - change_id = cache.get( - f"next_change_id:{self.leagues[0]["name"]}" - ) + change_id = cache.get(f"next_change_id:{self.leagues[0].name}") if change_id is None: change_id = self.initial_change_id # Make sure second listener also resets @@ -202,7 +144,11 @@ def _follow_stream( reset_event.clear() else: if sent_outgoing: - change_id = incoming.get() + try: + # The timeout is to avoid getting stuck when the other listener has stopped + change_id = incoming.get(timeout=10) + except Empty: + continue sent_outgoing = False @@ -330,9 +276,9 @@ def initialize_data_stream_threads( return futures @sync_timing_tracker - def _read_stream(self) -> tuple[list[Any], str | None]: + def _read_stream(self) -> tuple[list[Stash], str | None]: i = 0 - stashes = [] + stashes = list[Stash]() next_change_id = None while i < self.mini_batch_size: try: @@ -349,7 +295,7 @@ def _read_stream(self) -> tuple[list[Any], str | None]: continue obj = json.loads(pending.response.decode("utf-8")) - stashes.extend(obj["stashes"]) + stashes.extend(self.stashes_adapter.validate_python(obj["stashes"])) self.response_queue.task_done() next_change_id = pending.next_change_id @@ -358,19 +304,14 @@ def _read_stream(self) -> tuple[list[Any], str | None]: return stashes, next_change_id - @sync_timing_tracker - def _process_stream(self, stashes: list) -> pd.DataFrame: - logger.info("Stashes are ready for processing") - wanted_df = self._detector_filter(stashes) - logger.info("Finished processing the data, waiting for more") - return wanted_df - - def _gather_n_checkpoints(self, n: int) -> tuple[pd.DataFrame | None, str | None]: - df = None + def _gather_n_checkpoints(self, n: int) -> tuple[OrganizedItemsByCategory, str]: + all_organized_items = OrganizedItemsByCategory() for _ in range(n): start_time = time.perf_counter() stashes, next_change_id = self._read_stream() - wanted_df = self._process_stream(stashes) + organized_items = self.detector_controller.filter_stashes( + stashes, self.redis_cache + ) end_time = time.perf_counter() time_per_mini_batch = end_time - start_time @@ -383,27 +324,23 @@ def _gather_n_checkpoints(self, n: int) -> tuple[pd.DataFrame | None, str | None # Does not allow a batch to take longer than 2 minutes raise ProgramTooSlowException - if wanted_df.empty: - continue - - if df is None: - df = wanted_df - else: - df = pd.concat((df, wanted_df)) + all_organized_items.extend(organized_items) - return df, next_change_id + return all_organized_items, next_change_id - def dump_stream(self) -> Iterator[tuple[pd.DataFrame, str | None]]: - time.sleep(5) # Waits for the listening threads to have time to start up. + def dump_stream( + self, cache: redis.Redis + ) -> Iterator[tuple[OrganizedItemsByCategory, str]]: + self.redis_cache = cache + # time.sleep(5) # Waits for the listening threads to have time to start up. while True: logger.info("Waiting for data from the stream") - df, next_change_id = self._gather_n_checkpoints( + organized_items, next_change_id = self._gather_n_checkpoints( n=settings.N_CHECKPOINTS_PER_TRANSFORMATION, ) - if df is None: + if organized_items.is_empty(): logger.info("Found no data") continue logger.info("Finished processing the stream, entering transformation phase") - yield df.reset_index(), next_change_id - del df + yield organized_items, next_change_id logger.info("Finished transformation phase") diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/__init__.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py new file mode 100644 index 00000000..7843e04f --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/base.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel, ConfigDict + + +class PoeModel(BaseModel): + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + ) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py new file mode 100644 index 00000000..fe1d846c --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/currency.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + + +class CurrencyModel(BaseModel): + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + ) + + +class ExchangeRatioSide(BaseModel): + chaosValue: float | None = None + + +class ExchangeRatioItem(BaseModel): + id: int + + name: str + category: str + leagueId: int # supplied internally from DB + + chaos: ExchangeRatioSide + divine: ExchangeRatioSide diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py new file mode 100644 index 00000000..5a1dcafa --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/external/poe.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from pydantic import Field + +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.base import ( + PoeModel, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItem, + CacheStash, +) + +if TYPE_CHECKING: + # To avoid circular imports, while maintaining type hinting + from data_retrieval_app.external_data_retrieval.detectors.detector_controller import ( + DetectorController, + ) + + +class ItemModFlags(PoeModel): + fractured: bool | None = None + mutated: bool | None = None + crafted: bool | None = None + desecrated: bool | None = None + vestigial: bool | None = None + + +class ItemMod(PoeModel): + description: str + flags: ItemModFlags | None = None + + +class Extended(PoeModel): + prefixes: int | None = None + suffixes: int | None = None + + +class PoeItem(PoeModel): + # Basic identity / display + id: str | None = None + name: str + type_line: str = Field(alias="typeLine") + base_type: str = Field(alias="baseType") + rarity: str | None = None + + # PoeItem state + verified: bool + identified: bool + corrupted: bool | None = None + duplicated: bool | None = None + split: bool | None = None + fractured: bool | None = None + synthesised: bool | None = None + elder: bool | None = None + shaper: bool | None = None + searing: bool | None = None + tangled: bool | None = None + abyss_jewel: bool | None = Field( + default=None, + alias="abyssJewel", + ) + delve: bool | None = None + is_relic: bool | None = Field(default=None, alias="isRelic") + replica: bool | None = None + ruthless: bool | None = None + + # Level / league + league: str | None = None + ilvl: int + item_level: int | None = Field(default=None, alias="itemLevel") + monster_level: int | None = Field(default=None, alias="monsterLevel") + + # Inventory dimensions / location + w: int + h: int + x: int | None = None + y: int | None = None + inventory_id: str | None = Field( + default=None, + alias="inventoryId", + ) + + # Stack information + stack_size: int | None = Field(default=None, alias="stackSize") + max_stack_size: int | None = Field( + default=None, + alias="maxStackSize", + ) + stack_size_text: str | None = Field( + default=None, + alias="stackSizeText", + ) + + # Visuals + icon: str + art_filename: str | None = Field( + default=None, + alias="artFilename", + ) + frame_type: int | None = Field( + default=None, + alias="frameType", + ) + frame_type_id: str | None = Field( + default=None, + alias="frameTypeId", + ) + + # Mods + implicit_mods: list[ItemMod] | None = Field( + default=None, + alias="implicitMods", + ) + explicit_mods: list[ItemMod] | None = Field( + default=None, + alias="explicitMods", + ) + crafted_mods: list[str] | None = Field( + default=None, + alias="craftedMods", + ) + fractured_mods: list[str] | None = Field( + default=None, + alias="fracturedMods", + ) + enchant_mods: list[str] | None = Field( + default=None, + alias="enchantMods", + ) + scourge_mods: list[str] | None = Field( + default=None, + alias="scourgeMods", + ) + crucible_mods: list[str] | None = Field( + default=None, + alias="crucibleMods", + ) + veiled_mods: list[str] | None = Field( + default=None, + alias="veiledMods", + ) + cosmetic_mods: list[str] | None = Field( + default=None, + alias="cosmeticMods", + ) + utility_mods: list[str] | None = Field( + default=None, + alias="utilityMods", + ) + + # Categories / misc + category: dict[str, list[str]] | None = None + flavour_text: list[str] | None = Field( + default=None, + alias="flavourText", + ) + descr_text: str | None = Field( + default=None, + alias="descrText", + ) + sec_descr_text: str | None = Field( + default=None, + alias="secDescrText", + ) + note: str | None = None + forum_note: str | None = None + + # Special item data + extended: Extended | None = None + + # Other flags + locked_to_character: bool | None = Field( + default=None, + alias="lockedToCharacter", + ) + locked_to_account: bool | None = Field( + default=None, + alias="lockedToAccount", + ) + + veiled: bool | None = None + foreseeing: bool | None = None + + def to_cache(self, detector: DetectorController) -> CacheItem: + return CacheItem( + id=self.id, note=self.note, category=detector.get_category(self) + ) + + def __eq__(self, other: CacheItem | object) -> bool: + if isinstance(other, CacheItem): + print(other) + return other.id == self.id and other.note == self.note + return super().__eq__(other) + + def has_price(self) -> bool: + return self.note is not None and re.match( + r"^(~b\/o|~price) [0-9]*[.]?[0-9]+ [^ ]*$", self.note + ) + + +class Stash(PoeModel): + id: str + + public: bool + + stash: str | None = None + stash_type: str = Field(alias="stashType") + league: str | None = None + + items: list[PoeItem] = Field(default_factory=list) + + def to_cache(self, detector: DetectorController) -> CacheStash: + return CacheStash( + league=self.league, items=[item.to_cache(detector) for item in self.items] + ) + + def has_price(self) -> bool: + return self.stash is not None and re.match( + r"^(~b\/o|~price) \d+ [^ ]*$", self.stash + ) + + def get_item(self, id: str) -> tuple[int, PoeItem | None]: + for i, item in enumerate(self.items): + if item.id == id: + return i, item + + # this will crash the program if the idx is used + return len(self.items), None diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py new file mode 100644 index 00000000..561c4d8d --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/cache.py @@ -0,0 +1,18 @@ +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeModel, +) + + +class CacheItem(PoeModel): + id: str + note: str + category: str + + +class CacheItemWithContext(CacheItem): + league: str + + +class CacheStash(PoeModel): + league: str + items: list[CacheItem] diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py new file mode 100644 index 00000000..3810a93a --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/schemas/internal/categorized.py @@ -0,0 +1,47 @@ +from pydantic import Field + +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.base import ( + PoeModel, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItemWithContext, +) + + +class OrganizedItems(PoeModel): + removed_items: list[CacheItemWithContext] = Field(default_factory=list) + changed_items: list[PoeItem] = Field(default_factory=list) + new_items: list[PoeItem] = Field(default_factory=list) + + def extend(self, table: "OrganizedItems"): + self.removed_items.extend(table.removed_items) + self.changed_items.extend(table.changed_items) + self.new_items.extend(table.new_items) + + def is_empty(self) -> bool: + return not (self.removed_items or self.changed_items or self.new_items) + + +class OrganizedItemsByCategory(PoeModel): + """ + Field names much match the identifiers of the category detectors + """ + + unique: OrganizedItems = Field(default_factory=OrganizedItems) + unidentified_unique: OrganizedItems = Field(default_factory=OrganizedItems) + + def extend(self, categorized_items: "OrganizedItemsByCategory"): + self.unique.extend(categorized_items.unique) + self.unidentified_unique.extend(categorized_items.unidentified_unique) + + def is_empty(self) -> bool: + for table_name in self.model_fields: + table: OrganizedItems = getattr(self, table_name) + is_empty = table.is_empty() + if not is_empty: + return is_empty + + return True diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py index df4bb964..75d97afa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py @@ -1,151 +1,59 @@ -import inspect -import time -from typing import Any +from abc import ABC, abstractmethod -import pandas as pd +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, + Stash, +) -from data_retrieval_app.logs.logger import data_retrieval_logger as logger +class SubCategoryDetectorBase(ABC): + wanted_items: list[str] -class DetectorBase: + def is_wanted(self, item: PoeItem) -> bool: + return item.name in self.wanted_items + + @abstractmethod + def __str__(self): + """Returns a descriptive name for the sub category""" + + +class CategoryDetectorBase(ABC): """ Base class for searching stashes for items we want to store and process further """ - wanted_items = {} - found_items = {} - - def __init__( - self, leagues: list[dict[str, Any]], enable_pbar: bool = False - ) -> None: - """ - `self.n_unique_items_found` needs to be stored inbetween item detector sessions. - """ - self.n_unique_items_found = 0 - self.leagues = [league["name"] for league in leagues] - - self.prev_item_hashes_found = {} - - self.pbar_enabled = enable_pbar - - def _general_filter(self, df: pd.DataFrame) -> pd.DataFrame: - """ - Filtering away items that are never useful - """ - columns = df.columns - if "ruthless" in columns: - df = df.loc[df["ruthless"].isnull()] - if "lockedToCharacter" in columns: - df = df.loc[df["lockedToCharacter"].isnull()] - if "lockedToAccount" in columns: - df = df.loc[df["lockedToAccount"].isnull()] - if "logbookMods" in columns: - df = df.loc[df["logbookMods"].isnull()] - if "crucible" in columns: - df = df.loc[df["crucible"].isnull()] - if "scourged" in columns: - df = df.loc[df["scourged"].isnull()] - if "hybrid" in columns: - df = df.loc[df["hybrid"].isnull()] - if "ultimatumMods" in columns: - df = df.loc[df["ultimatumMods"].isnull()] - if "stash" in columns or "note" in columns: - if "stash" in columns and "note" in columns: - df = df.loc[ - df["stash"].str.startswith(("~b/o", "~price")) - | df["note"].str.startswith(("~b/o", "~price")) - ] - elif "stash" in columns: - df = df.loc[df["stash"].str.startswith(("~b/o", "~price"))] - else: - df = df.loc[df["note"].str.startswith(("~b/o", "~price"))] - else: - return pd.DataFrame(columns=df.columns) - - df = df.loc[df["league"].isin(self.leagues)] - - return df - - def _specialized_filter(self, df: pd.DataFrame) -> pd.DataFrame: - raise NotImplementedError( - "This method is implemented in child classes. Do not use the base on its own." - ) - - def _filter_on_game_item_id(self, df: pd.DataFrame) -> pd.DataFrame: - """ - The same item can be picked up by the api, without any changes being made to the listing/item, - because the user listed/changed another item in the same tab. We do not want to store these, - as they pain the wrong picture of the market. - - To make sure that only updated items are kept, we also add the note, which contains the price. - Please note that this does not consider relisting of entire tab. - - However, as of right now, these identifiers reset every ~hour. In the future we may pull - the most recent listings from the database instead of resetting. - """ - n_items_before_filter = len(df) - - if "note" in df.columns: - df = df.drop_duplicates(["id", "note"]) - note = df["note"].apply(hash) - else: - df = df.drop_duplicates("id") - note = 0 - - game_item_id = df["id"].apply(int, base=16) - - hashes: pd.Series[int] = game_item_id + note - - unique_hashes = set(hashes) - - if not self.prev_item_hashes_found: - self.prev_item_hashes_found = unique_hashes - else: - duplicate_hashes = self.prev_item_hashes_found.intersection(unique_hashes) - self.prev_item_hashes_found |= unique_hashes - - items_to_drop_mask = ~hashes.isin(duplicate_hashes) - df = df.loc[items_to_drop_mask] - - n_items_filtered = n_items_before_filter - len(df) - - logger.info( - f'detector="{self}" {n_items_before_filter=} {n_items_filtered=} percent_of_total={1 - n_items_filtered / max(1, n_items_before_filter):.2%}' - ) - - return df - - def _snapshot(self, df: pd.DataFrame, filepath: str = None) -> None: - """ - A method for creating a snapshot for a given dataframe. If no filepath is given, it saves the class - this directory with the name `snapshot_{name_of_class}_{time}.csv` - """ - if filepath is None: - filepath = ( - inspect.getfile(DetectorBase)[:-7] - + f"snapshot_{self}_{time.time():.0f}.csv" - ) - - logger.info(f"Saving a snapshot, for the detector {self}") - df.to_csv(filepath, encoding="utf-8") - - def iterate_stashes( - self, df: pd.DataFrame - ) -> tuple[pd.DataFrame, int, int, pd.DataFrame]: - """ - TODO - """ - - df = self._general_filter(df) - if df.empty: - return df, 0, self.n_unique_items_found, df - df_filtered = self._specialized_filter(df) - df_leftover = df.loc[~df.index.isin(df_filtered.index)] - - df_filtered = self._filter_on_game_item_id(df_filtered) - - item_count = len(df_filtered) - if self.pbar_enabled: - self.n_unique_items_found = len(self.found_items.keys()) - - return df_filtered, item_count, self.n_unique_items_found, df_leftover + identifier: str + + should_cache: bool + + detectors: list[SubCategoryDetectorBase] + + @abstractmethod + def matches_category(self, item: PoeItem) -> bool: + """Returns true or false based on the criteria of the detector""" + + def find_interesting_items( + self, donor_stash: Stash, receiver_stash: Stash, no_cache_receiver_stash: Stash + ) -> bool: + donated_items = list[int]() + for i, item in enumerate(donor_stash.items): + if not self.matches_category(item): + continue + + is_wanted = False + for detector in self.detectors: + is_wanted = detector.is_wanted(item) + if is_wanted: + break + + if is_wanted: + if self.should_cache: + receiver_stash.items.append(item) + else: + no_cache_receiver_stash.items.append(item) + donated_items.append(i) + + for idx in reversed(donated_items): + donor_stash.items.pop(idx) + + return len(donated_items) > 0 diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py new file mode 100644 index 00000000..9d5370dd --- /dev/null +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/detector_controller.py @@ -0,0 +1,461 @@ +import json +from typing import Any + +import redis +from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter + +from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, + Stash, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItem, + CacheItemWithContext, + CacheStash, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.categorized import ( + OrganizedItems, + OrganizedItemsByCategory, +) +from data_retrieval_app.external_data_retrieval.detectors.base import ( + CategoryDetectorBase, +) +from data_retrieval_app.external_data_retrieval.utils import ( + sync_timing_tracker, +) +from data_retrieval_app.logs.logger import data_retrieval_logger as logger + + +class DetectorController: + backend_base_url = settings.BACKEND_BASE_URL + existing_items_url = f"{backend_base_url}/item/existing/" + + def __init__( + self, + detectors: list[CategoryDetectorBase], + leagues: list[League], + ): + self.leagues = [league.name for league in leagues] + + self.detectors = detectors + + self.local_cache = dict[str, CacheStash]() + self.cached_stash_adapter = TypeAdapter(CacheStash) + + def _filter_duplicates(self, stashes: list[Stash]) -> list[Stash]: + previous_ids = list[str]() + non_duplicate_stashes = list[Stash]() + duplicates_found = 0 + for stash in reversed(stashes): + if stash.id in previous_ids: + duplicates_found += 1 + continue + + previous_ids.append(stash.id) + non_duplicate_stashes.append(stash) + + logger.info( + f"task=filter_duplicates n_stashes_in={len(stashes)} n_duplicate_stashes={duplicates_found}" + ) + return non_duplicate_stashes + + def _filter_never_used( + self, stashes: list[Stash] + ) -> tuple[list[Stash], list[Stash]]: + uninteresting_item_fields = { + "monsterLevel", + "lockedToCharacter", + "lockedToAccount", + "logbookMods", + "ultimatumMods", + "mercenarySkills", + "isRelic", + "enshrouded", + "ruthless", + "hybrid", + "unmodifiableExceptChaos", + } + + filtered_stashes = list[Stash]() + empty_stashes = list[Stash]() + + n_uninteresting_items = 0 + n_items_in = 0 + + for stash in stashes: + items: list[PoeItem] = stash.items + filtered_items = [] + if not stash.public or stash.league not in self.leagues: + # TODO should I already filter leagues? depends on what happens on migration + continue + n_items_in += len(items) + for item in items: + if ( + uninteresting_item_fields + & item.model_dump(exclude_none=True).keys() + ): + # if any of these fields are present, it automatically means we are not interested + n_uninteresting_items += 1 + continue + + if item.has_price(): + pass + elif stash.has_price(): + item.note = stash.stash + else: + n_uninteresting_items += 1 + continue + + filtered_items.append(item) + + if filtered_items: + stash.items = filtered_items + filtered_stashes.append(stash) + else: + stash.items = [] + empty_stashes.append(stash) + + logger.info( + f"task=filter_never_used n_stashes_in={len(stashes)} n_stashes_found={len(filtered_stashes)} n_items_in={n_items_in} n_items_filtered={n_uninteresting_items}" + ) + return filtered_stashes, empty_stashes + + def _filter_uninteresting_items( + self, stashes: list[Stash] + ) -> tuple[list[Stash], list[PoeItem], list[Stash]]: + n_uninteresting_items = 0 + n_items_in = 0 + + interesting_stashes = list[Stash]() + no_cache_new_items = list[PoeItem]() + filtered_stashes = list[Stash]() + for donor_stash in stashes: + n_items_in += len(donor_stash.items) + receiver_stash = donor_stash.model_copy(update={"items": []}) + no_cache_receiver_stash = donor_stash.model_copy(update={"items": []}) + found_items = False + for detector in self.detectors: + # the donor stash is modified inplace, with its items moving over to the receiver stashes inplace + found_items = ( + detector.find_interesting_items( + donor_stash, receiver_stash, no_cache_receiver_stash + ) + or found_items + ) + + if found_items: + if receiver_stash.items: + interesting_stashes.append(receiver_stash) + + if no_cache_receiver_stash.items: + # items such as unidentified items + no_cache_new_items.extend(no_cache_receiver_stash.items) + else: + n_uninteresting_items += len(donor_stash.items) + + filtered_stashes.append(donor_stash) + + logger.info( + f"task=filter_uninteresting n_stashes_in={len(stashes)} n_stashes_filtered={len(filtered_stashes)} n_items_in={n_items_in} n_items_filtered={n_uninteresting_items}" + ) + return interesting_stashes, no_cache_new_items, filtered_stashes + + def _get_cached_stashes( + self, redis_cache: redis.Redis, ids: list[str] + ) -> list[CacheStash | None]: + """ + First checks local cache, then redis cache. This method relies on the local cache + being exactly aligned with the redis cache for stashes that exists in both. + """ + if len(ids) == 0: + return [] + + # first local cache + cached_stashes = list[CacheStash | None]() + still_needs_check = list[tuple[int, str]]() + for i, id in enumerate(ids): + cached_stash = self.local_cache.get(f"stash:{id}") + if cached_stash is None: + still_needs_check.append((i, id)) + cached_stashes.append(None) + else: + cached_stashes.append( + self.cached_stash_adapter.validate_json(cached_stash) + ) + + # then redis cache + redis_cached_stashes = redis_cache.mget( + [f"stash:{id}" for _, id in still_needs_check] + ) + for i, cached_stash in enumerate(redis_cached_stashes): + if cached_stash is not None: + idx, _ = still_needs_check[i] + cached_stashes[idx] = self.cached_stash_adapter.validate_json( + cached_stash + ) + + return cached_stashes + + def _find_removed_items( + self, empty_stash_ids: list[str], redis_cache: redis.Redis + ) -> list[CacheItemWithContext]: + # TODO what if league can change but id stays the same + empty_cached_stashes = self._get_cached_stashes(redis_cache, empty_stash_ids) + removed_items = list[CacheItemWithContext]() + cached_stashes_to_remove = list[str]() + + for id, cached_stash in zip(empty_stash_ids, empty_cached_stashes, strict=True): + if cached_stash is not None: + cached_stashes_to_remove.append(f"stash:{id}") + for cached_item in cached_stash.items: + removed_items.append( + CacheItemWithContext( + **cached_item.model_dump(), league=cached_stash.league + ) + ) + + if cached_stashes_to_remove: + [self.local_cache.pop(id, None) for id in cached_stashes_to_remove] + redis_cache.delete(*cached_stashes_to_remove) + + logger.info( + f"task=find_removed n_stashes_in={len(empty_stash_ids)} n_stashes_cached={len(cached_stashes_to_remove)} n_removed_items={len(removed_items)}" + ) + return removed_items + + def _find_changed_items_filter_unchanged( + self, stashes: list[Stash], redis_cache: redis.Redis + ) -> tuple[list[CacheItemWithContext], list[PoeItem], list[PoeItem]]: + # TODO what if league can change but id stays the same + not_empty_stash_ids = [stash.id for stash in stashes] + + not_empty_cached_stashes = self._get_cached_stashes( + redis_cache, not_empty_stash_ids + ) + + changed_items = list[PoeItem]() + new_items = list[PoeItem]() + removed_items = list[CacheItemWithContext]() + + n_stashes_cached = 0 + n_items_filtered = 0 + + stashes_to_cache = dict[str, CacheStash]() + for cached_stash, stash in zip(not_empty_cached_stashes, stashes, strict=True): + cached_stash_changed = False + if cached_stash is not None: + n_stashes_cached += 1 + for cached_item in cached_stash.items[:]: + idx, new_item = stash.get_item(cached_item.id) + if new_item is None: + # a cached item doesnt exist anymore + cached_stash_changed = True + cached_stash.items.remove(cached_item) + removed_items.append( + CacheItemWithContext( + id=cached_item.id, + note=cached_item.note, + category=cached_item.category, + league=stash.league, + ) + ) + continue + if new_item.note == cached_item.note: + # Duplicate found in cache + n_items_filtered += 1 + stash.items.pop(idx) + continue + + # PoeItem has changed from cache + changed_items.append(new_item) + cached_stash_changed = True + cached_stash.items.remove(cached_item) + + cached_stash.items.extend(stash.to_cache(self).items) + + else: + # The stash has never been cached + cached_stash_changed = True + cached_stash = stash.to_cache(self) + + new_items.extend(stash.items) + + if cached_stash_changed and cached_stash.items: + stashes_to_cache[f"stash:{stash.id}"] = cached_stash.model_dump_json() + + if stashes_to_cache: + self.local_cache.update(stashes_to_cache) + redis_cache.mset(stashes_to_cache) + + logger.info( + f"task=find_changed_filter_unchanged n_stashes_in={len(stashes)} n_stashes_cached={n_stashes_cached} n_items_filtered={n_items_filtered} n_removed_items={len(removed_items)} n_changed_items={len(changed_items)} n_new_items={len(new_items)}" + ) + return removed_items, changed_items, new_items + + def _organize_items( + self, + removed_items: list[CacheItemWithContext], + changed_items: list[PoeItem], + new_items: list[PoeItem], + ) -> OrganizedItemsByCategory: + categorized_items = OrganizedItemsByCategory() + for detector in self.detectors: + table = OrganizedItems() + + for removed_item in removed_items[:]: + if detector.identifier == removed_item.category: + table.removed_items.append(removed_item) + removed_items.remove(removed_item) + + for changed_item in changed_items[:]: + if detector.matches_category(changed_item): + table.changed_items.append(changed_item) + changed_items.remove(changed_item) + + for new_item in new_items[:]: + if detector.matches_category(new_item): + table.new_items.append(new_item) + new_items.remove(new_item) + + setattr(categorized_items, detector.identifier, table) + + return categorized_items + + def get_category(self, item: PoeItem) -> str: + for detector in self.detectors: + if detector.matches_category(item): + return detector.identifier + + raise ValueError("PoeItem does not match a detector category") + + @sync_timing_tracker + def filter_stashes( + self, stashes: list[Stash], redis_cache: redis.Redis + ) -> OrganizedItemsByCategory: + """ + Parameters: + :param stashes: a list of stash objects + :param redis_cache: a redis cache + Returns: + :Removed items: items which need to be closed + :Changed items: items which need to start a new entry + :New items: items which need to be entered into the system + """ + stashes = self._filter_duplicates(stashes) + + filtered_stashes, empty_stashes = self._filter_never_used(stashes) + empty_stash_ids = [stash.id for stash in empty_stashes] + + ( + interesting_stashes, + no_cache_new_items, + uninteresting_stashes, + ) = self._filter_uninteresting_items(filtered_stashes) + empty_stash_ids.extend([stash.id for stash in uninteresting_stashes]) + + removed_items = self._find_removed_items(empty_stash_ids, redis_cache) + + ( + more_removed_items, + changed_items, + more_new_items, + ) = self._find_changed_items_filter_unchanged(interesting_stashes, redis_cache) + + new_items = no_cache_new_items + more_new_items + removed_items.extend(more_removed_items) + + logger.info( + f"task=total n_removed_items={len(removed_items)} n_changed_items={len(changed_items)} n_new_items={len(new_items)}" + ) + + return self._organize_items(removed_items, changed_items, new_items) + + +if __name__ == "__main__": + from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( + UnidentifiedUniqueDetector, + UniqueDetector, + ) + + class RedisCacheDecoy: + cache = { + "3ca7f08b3382795478397c41d110fc407da007bffd7e88dd9e6501bb306c7a6a": { + # must be removed + "league": "Standard", + "items": [ + { # doesnt exist anymore + "id": "1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402", + "note": "~b/o 30 chaos", + } + ], + }, + "eca1a8b071f18a3f07aa55f914173287a445f46b8e433ed3e04edf636dc6b946": { + "league": "Allflame", + "items": [ + { # duplicate + "id": "5588417cffcfbd0efe651cde0e5c606867d2b58a34812b90ce78e1352bacf68a", + "note": "~b/o 32 divine", + } + ], + }, + } + + def mget(self, keys: list[str]): + return [self.cache.get(key) for key in keys] + + def mset(self, key_to_object: dict[str, Any]): + for key, obj in key_to_object.items(): + self.cache[key] = obj + + def delete(self, keys: list[str]): + for key in keys: + self.cache.pop(key, None) + + detector_controller = DetectorController( + [ + UniqueDetector(), + UnidentifiedUniqueDetector(), + ], + [{"name": "Standard"}, {"name": "Hardcore"}, {"name": "Allflame"}], + redis_cache=RedisCacheDecoy(), + ) + + detector_controller.local_cache[ + "158c93260022f9b39f9e7c6980711ab057c6ef2e28e1cdf00f2c2b5aef2c880c" + ] = CacheStash( + league="Standard", + items=[ + CacheItem( # changed price + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_updated", + note="~b/o 2 mirror", + ), + CacheItem( # same price + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402", + note="~b/o 30 chaos", + ), + CacheItem( # removed + id="1b6b82e172a31022acdc712dd59c641db1ce741352bdbcf967880e10f135e402_removed", + note="~b/o 2 mirror", + ), + ], + ) + + stashes = json.load(open("stashes.json")) + stashes = [ + stash + for stash in stashes + if stash["id"] + in ( + "b3e32cdca2367d795b6d37b625bf22096ea2ee405b20357dc80c406dd0e4b63e", # empty + "ea0a95a395b48875e81f947cc87e315ba4a719fb08a5453f70c9e8c6fd2d0266", # uninteresting + "a68ead45e8bf786cec6c52d37a226739b5797f1c0264cf64321635f448dac97e", # interesting + "158c93260022f9b39f9e7c6980711ab057c6ef2e28e1cdf00f2c2b5aef2c880c", # interesting + "3ca7f08b3382795478397c41d110fc407da007bffd7e88dd9e6501bb306c7a6a", # uninteresting + "eca1a8b071f18a3f07aa55f914173287a445f46b8e433ed3e04edf636dc6b946", # interesting + ) + ] + stash_adapter = TypeAdapter(list[Stash]) + stashes = stash_adapter.validate_python(stashes) + detector_controller.filter_stashes(stashes) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py index 11820354..b59d19cb 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/unique_detector.py @@ -1,31 +1,91 @@ -import pandas as pd +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, +) +from data_retrieval_app.external_data_retrieval.detectors.base import ( + CategoryDetectorBase, + SubCategoryDetectorBase, +) -from data_retrieval_app.external_data_retrieval.detectors.base import DetectorBase +class UniqueJewelDetector(SubCategoryDetectorBase): + wanted_items = [ + "Grand Spectrum", + "Forbidden Flesh", + "Forbidden Flame", + "The Balance of Terror", + "That Which Was Taken", + "Split Personality", + "Thread of Hope", + "The Adorned", + "Impossible Escape", + "Watcher's Eye", + "Bound By Destiny", + "Sublime Vision", + "The Light of Meaning", + "Glorious Vanity", + "Lethal Pride", + "Brutal Restraint", + "Militant Faith", + "Elegant Hubris", + "Voices", + "Heroic Tragedy", + ] -class UniqueDetector(DetectorBase): - def _check_if_wanted(self, df: pd.DataFrame) -> pd.DataFrame: - df = df.loc[df["name"].isin(self.wanted_items)] - return df + def __str__(self): + return "Unique Jewel Detector" - def _specialized_filter(self, df: pd.DataFrame) -> pd.DataFrame: - if "rarity" not in df.columns: - return pd.DataFrame(columns=df.columns) - df = df.loc[df["rarity"] == "Unique"] +class UniqueJewelleryDetector(SubCategoryDetectorBase): + wanted_items = [ + "Aul's Uprising", + "Replica Dragonfang's Flight", + "The Utmost", + "Precursor's Emblem", + "Circle of Ambition", + "Circle of Guilt", + "Circle of Anguish", + "Circle of Regret", + "Circle of Fear", + "Circle of Nostalgia", + "Screams of the Desiccated", + ] + + def __str__(self): + return "Unique Jewellery Detector" + + +class UniqueArmourDetector(SubCategoryDetectorBase): + wanted_items = ["Forbidden Shako", "Skin of the Lords", "Shroud of the Lightless"] + + def __str__(self): + return "Unique Armour Detector" + + +class UniqueWeaponDetector(SubCategoryDetectorBase): + wanted_items = ["Paradoxica", "Cane of Kulemak"] + + def __str__(self): + return "Unique Weapon Detector" - df = self._check_if_wanted(df) - if self.pbar_enabled: - temp_df = df["name"] + df["baseType"] - for name_baseType in temp_df.unique(): - if name_baseType not in self.found_items: - self.found_items[name_baseType] = True +class UniqueDetector(CategoryDetectorBase): + identifier = "unique" - return df + should_cache = True + + detectors = [ + UniqueJewelDetector(), + UniqueJewelleryDetector(), + UniqueArmourDetector(), + UniqueWeaponDetector(), + ] + def matches_category(self, item: PoeItem) -> bool: + return item.identified and item.rarity is not None and item.rarity == "Unique" -class UniqueUnidentifiedDetector(UniqueDetector): + +class UnidentifiedUniqueDetector(CategoryDetectorBase): + # TODO separate this into sub categories """ Notes: Precursor's Emblems, Shroud of the Lightless and Paradoxica are not supported. @@ -40,6 +100,10 @@ class UniqueUnidentifiedDetector(UniqueDetector): No, because they are not particularly sought after in their un-id form """ + identifier = "unidentified_unique" + + should_cache = False + wanted_base_types = [ "Viridian Jewel", "Cobalt Jewel", @@ -96,81 +160,24 @@ class UniqueUnidentifiedDetector(UniqueDetector): "Soulcord.png": "Screams of the Desiccated", } - def _check_if_wanted(self, df: pd.DataFrame) -> pd.DataFrame: + def matches_category(self, item: PoeItem) -> bool: """ - Uses the icon to identify which unique it is, then saving that name. - If the name attribute still has a length of 0 it means no matching unique - was found. + Uses the icon to identify which unique it is, then saving that name inplace. """ - df = df.loc[(~df["identified"] & df["baseType"].isin(self.wanted_base_types))] + if ( + item.identified + or (item.rarity is None or item.rarity != "Unique") + or item.base_type not in self.wanted_base_types + ): + return False - for icon, name in self.wanted_item_icons.items(): - df.loc[df["icon"].str.endswith(icon), "name"] = name + icon = item.icon.split("/")[-1] + name = self.wanted_item_icons.get(icon) + if name is None: + return False - df = df.loc[df["name"].str.len() != 0] - # self._snapshot(df) - return df + item.name = name + return True def __str__(self): return "Unidentifed Unique detector" - - -class UniqueJewelDetector(UniqueDetector): - wanted_items = [ - "Grand Spectrum", - "Forbidden Flesh", - "Forbidden Flame", - "The Balance of Terror", - "That Which Was Taken", - "Split Personality", - "Thread of Hope", - "The Adorned", - "Impossible Escape", - "Watcher's Eye", - "Bound By Destiny", - "Sublime Vision", - "The Light of Meaning", - "Glorious Vanity", - "Lethal Pride", - "Brutal Restraint", - "Militant Faith", - "Elegant Hubris", - "Voices", - "Heroic Tragedy", - ] - - def __str__(self): - return "Unique Jewel Detector" - - -class UniqueJewelleryDetector(UniqueDetector): - wanted_items = [ - "Aul's Uprising", - "Replica Dragonfang's Flight", - "The Utmost", - "Precursor's Emblem", - "Circle of Ambition", - "Circle of Guilt", - "Circle of Anguish", - "Circle of Regret", - "Circle of Fear", - "Circle of Nostalgia", - "Screams of the Desiccated", - ] - - def __str__(self): - return "Unique Jewellery Detector" - - -class UniqueArmourDetector(UniqueDetector): - wanted_items = ["Forbidden Shako", "Skin of the Lords", "Shroud of the Lightless"] - - def __str__(self): - return "Unique Armour Detector" - - -class UniqueWeaponDetector(UniqueDetector): - wanted_items = ["Paradoxica", "Cane of Kulemak"] - - def __str__(self): - return "Unique Weapon Detector" diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py index 9dc4d74c..e58f2be6 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py @@ -6,11 +6,10 @@ ThreadPoolExecutor, wait, ) -from io import StringIO -from typing import Any -import pandas as pd import redis +from backend_api.app.core.schemas.league import League +from pydantic import TypeAdapter from data_retrieval_app.external_data_retrieval.cache import get_cache from data_retrieval_app.external_data_retrieval.config import settings @@ -20,12 +19,8 @@ from data_retrieval_app.external_data_retrieval.data_retrieval.poe_api_handler import ( PoEAPIHandler, ) -from data_retrieval_app.external_data_retrieval.transforming_data.transform_currency_api_data import ( - TransformCurrencyAPIData, -) from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_api_data import ( - PoEAPIDataTransformerBase, - UniquePoEAPIDataTransformer, + PoEAPIDataTransformer, ) from data_retrieval_app.external_data_retrieval.utils import ( ProgramTooSlowException, @@ -43,21 +38,14 @@ class ContinuousDataRetrieval: stash_tab_url = "https://api.pathofexile.com/public-stash-tabs" backend_base_url = settings.BACKEND_BASE_URL - modifier_url = f"{backend_base_url}/modifier/" active_league_url = f"{backend_base_url}/league/active_league/" - item_base_type_url = f"{backend_base_url}/itemBaseType/" currency_url = f"{backend_base_url}/currency/" pom_auth_headers = get_superuser_token_headers(backend_base_url) - def __init__( - self, - data_transformers: dict[str, PoEAPIDataTransformerBase], - ): + def __init__(self): self.leagues = self._get_leagues() - self.data_transformers: dict[str, PoEAPIDataTransformerBase] = { - key: data_transformer(self.leagues) - for key, data_transformer in data_transformers.items() - } + + self.data_transformer = PoEAPIDataTransformer(self.leagues) self.poe_api_handler = PoEAPIHandler( url=self.stash_tab_url, @@ -68,172 +56,37 @@ def __init__( self.currency_api_handler = CurrencyAPIHandler( url="https://api.poe.watch/exchange/ratios?league={league}&game=poe1" ) - self.currency_transformer = TransformCurrencyAPIData() - - def _get_modifiers(self) -> dict[str, pd.DataFrame]: - response = get_data_safe( - self.modifier_url, headers=self.pom_auth_headers, logger=logger - ) - # Check if the request was successful - modifier_df = pd.DataFrame() - # Load the JSON data into a pandas DataFrame - json_io = StringIO(response.content.decode("utf-8")) - modifier_df = pd.read_json(json_io, dtype=str) - - modifier_types = [ - "implicit", - "explicit", - "delve", - "fractured", - "synthesised", - "unique", - "corrupted", - "enchanted", - "veiled", - ] - modifier_dfs = {} - for modifier_type in modifier_types: - if modifier_type in modifier_df.columns: - modifier_dfs[modifier_type] = modifier_df.loc[ - ~modifier_df[modifier_type].isna() - ] - return modifier_dfs - def _get_leagues(self) -> list[dict[str, Any]]: + def _get_leagues(self) -> list[League]: response = get_data_safe( self.active_league_url, headers=self.pom_auth_headers, logger=logger ) - leagues = response.json() - - return leagues - - def _get_item_base_types(self) -> dict[str, int]: - response = get_data_safe( - self.item_base_type_url, headers=self.pom_auth_headers, logger=logger - ) - item_base_type_mapped = {} - item_base_types = [] - - item_base_types = response.json() - if not isinstance(item_base_types, list): - item_base_types = [item_base_types] - for item_base_type in item_base_types: - item_base_type_id = item_base_type["itemBaseTypeId"] - base_type = item_base_type["baseType"] - item_base_type_mapped[base_type] = item_base_type_id - - return item_base_type_mapped - - def _categorize_new_items(self, df: pd.DataFrame) -> dict[str, pd.DataFrame]: - split_dfs = {} - - # TODO not fully exhaustive yet, needs to be updated over time - # category_priority = [ - # "synthesised", - # "fractured", - # "delve", - # "veiled", - # "unique", - # ] - # Needs to take priority, see nebulis and rational doctrine - # not_synth_mask = df["synthesised"].isna() - # split_dfs["synthesised"] = df.loc[~not_synth_mask] - # df = df.loc[not_synth_mask] - - not_unique_mask = df["rarity"] != "Unique" - split_dfs["unique"] = df.loc[~not_unique_mask] - df = df.loc[not_unique_mask] - - # for category in category_priority: - # mask = df[category].isna() - - # split_dfs[category] = df.loc[~mask] - # df = df.loc[mask] - - return split_dfs - - def _get_currency_data(self, current_hours: dict[int, int]) -> pd.DataFrame: - league_ids = list(current_hours.keys()) - response = get_data_safe( - self.currency_url + "latest_hours/", - params={"league_ids": league_ids}, - headers=self.pom_auth_headers, - logger=logger, - ) - latest_hours: dict[str, int] = response.json() - need_new_data = [] - need_old_data = [] - if latest_hours: - for league_id, latest_hour in latest_hours.items(): - league_id = int(league_id) - if ( - league_id in current_hours - and latest_hour == current_hours[league_id] - ): - need_old_data.append(league_id) - else: - need_new_data.append(league_id) - else: - need_new_data = league_ids - - currency_df = None - if need_old_data: - response = get_data_safe( - self.currency_url + "latest_currencies/", - params={"league_ids": need_old_data}, - headers=self.pom_auth_headers, - logger=logger, - ) - - currency_df = pd.DataFrame(response.json()) - - if need_new_data: - needed_leagues = [ - league for league in self.leagues if league["leagueId"] in need_new_data - ] - new_data = self.currency_api_handler.make_request(needed_leagues) - new_data = self.currency_transformer.transform_into_tables( - new_data, current_hours - ) - if currency_df is None: - currency_df = new_data - else: - currency_df = pd.concat((currency_df, new_data)) - - return currency_df + return TypeAdapter(list[League]).validate_python(response.json()) def _follow_data_dump_stream(self, cache: redis.Redis): current_hours = find_hours_since_launch(self.leagues) # Only need to refer to one league to see when a new hour starts - current_hour = current_hours[self.leagues[0]["leagueId"]] + current_hour = current_hours[self.leagues[0].leagueId] next_hour = current_hour + 1 logger.info("Retrieving modifiers from db.") - modifier_dfs = self._get_modifiers() - item_base_types = self._get_item_base_types() - currency_df = self._get_currency_data(current_hours) - iter_data = self.poe_api_handler.dump_stream() + currencies = self.currency_api_handler.get_currency_data( + self.leagues, current_hours + ) + self.data_transformer.set_currencies(currencies) + self.data_transformer.set_current_hours(current_hours) + iter_data = self.poe_api_handler.dump_stream(cache) while current_hour < next_hour: - df, next_change_id = next(iter_data) - if df.empty: - continue - split_dfs = self._categorize_new_items(df) - for data_transformer_type in self.data_transformers: - self.data_transformers[data_transformer_type].transform_into_tables( - df=split_dfs[data_transformer_type], - modifier_df=modifier_dfs[data_transformer_type], - currency_df=currency_df.copy(deep=True), - item_base_types=item_base_types, - current_hours=current_hours, - ) + organized_items, next_change_id = next(iter_data) + self.data_transformer.transform_and_insert(organized_items) if next_change_id is not None: # Only set the next change id once the data has been safely inserted cache.set("next_change_id", next_change_id) current_hours = find_hours_since_launch(self.leagues) - current_hour = current_hours[self.leagues[0]["leagueId"]] - for data_transformer_type in self.data_transformers: - self.data_transformers[data_transformer_type].end_of_hour_cleanup() + current_hour = current_hours[self.leagues[0].leagueId] + + self.data_transformer.end_of_hour_cleanup() def retrieve_data(self): logger.info("Program starting up.") @@ -318,11 +171,8 @@ def retrieve_data(self): def main(): logger.info("Starting the program...") setup_logging() - data_transformers = {"unique": UniquePoEAPIDataTransformer} - data_retriever = ContinuousDataRetrieval( - data_transformers=data_transformers, - ) + data_retriever = ContinuousDataRetrieval() data_retriever.retrieve_data() diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py index 92828add..d88b7444 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/roll_processor.py @@ -1,201 +1,100 @@ import re - -import pandas as pd - +from collections import defaultdict + +from backend_api.app.core.schemas.item_modifier import ( + ItemModifierCreate, + ItemModifierRoll, +) +from backend_api.app.core.schemas.modifier import GroupedModifier + +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + ItemMod, + PoeItem, +) from data_retrieval_app.logs.logger import transform_logger as logger -pd.set_option("display.max_colwidth", None) - class RollProcessor: - @property - def modifier_df(self) -> pd.DataFrame: - return self._modifier_df - - @modifier_df.setter - def modifier_df(self, modifier_df: pd.DataFrame): - self._modifier_df = modifier_df.drop(["createdAt"], axis=1) - - static_modifier_mask = self._modifier_df["static"] == "True" - self.static_modifier_df = self._modifier_df.loc[static_modifier_mask] - - self.dynamic_modifier_df = self._modifier_df.loc[~static_modifier_mask] - - def add_modifier_df(self, modifier_df: pd.DataFrame): - try: - modifier_df = self.modifier_df - except AttributeError: - self.modifier_df = modifier_df - - def _pre_processing(self, df: pd.DataFrame) -> pd.DataFrame: - no_modifiers_mask = df["modifier"].isna() - df = df.loc[~no_modifiers_mask] - df["modifier"] = df["modifier"].apply(lambda mod: mod["description"]) - df.loc[:, "modifier"] = df[ - "modifier" - ].replace( - r"\\n|\n", " ", regex=True - ) # Replaces newline with a space, so that it does not mess up the regex and matches modifiers in the `modifier` table - # Removes all rows with no modifier (The Adorned) - - return df - - def _process_static( - self, df: pd.DataFrame, static_modifers_mask: pd.Series - ) -> pd.DataFrame: - """ - Static modifiers must be processed first, to reduce the amount of modifiers - processed by the much more expensive dynamic modifier processing. - """ - static_modifier_df = self.static_modifier_df - - static_df = df.loc[static_modifers_mask] - if static_df.empty: - return pd.DataFrame( - columns=static_df.columns.append(static_modifier_df.columns) - ) - static_df.loc[:, "position"] = "0" - static_df.loc[:, "effect"] = static_df.loc[:, "modifier"] - - merged_static_df = static_df.merge( - static_modifier_df, on=["effect", "position"], how="left" - ) - failed_df = merged_static_df.loc[merged_static_df["static"].isna()] - - if not failed_df.empty: - logger.debug( - f"Failed to merge static modifier with modifier in DB.\n{failed_df}" - ) - # remove all modifiers that failed to merge - # NOTE this should never happen - merged_static_df = merged_static_df.loc[~merged_static_df["static"].isna()] - - return merged_static_df - - def _get_rolls(self, dynamic_df: pd.DataFrame) -> pd.DataFrame: - """ - Uses regex matching groups to extract the rolls and adds - the correct effect. - """ - - def extract_rolls(matchobj: re.Match) -> str: - rolls = [ - roll - for roll in matchobj.groups() - if roll not in ["reduced", "increased"] # because of alternate spelling + def __init__(self, modifiers: dict[str, list[GroupedModifier]]): + self.modifiers = modifiers + + self.missing_modifiers = defaultdict[str, set[str]](set) + + def _process_dynamic_modifier( + self, db_mod: GroupedModifier, match: re.Match + ) -> list[ItemModifierRoll]: + position = 0 + rolls = list[ItemModifierRoll]() + for roll in match.groups(): + try: + if roll in ["reduced", "increased"]: + continue + + db_roll = db_mod.rolls[position] + if db_roll.textRolls is not None: + extracted_roll = db_roll.textRolls.index(roll) + else: + extracted_roll = float(roll) + + rolls.append(ItemModifierRoll(position=position, roll=extracted_roll)) + position += 1 + except: + print(roll, position, db_mod) + raise + + return rolls + + def _extract_rolls( + self, modifier: ItemMod, db_modifiers: list[GroupedModifier] + ) -> ItemModifierCreate | None: + # pre processing + effect = modifier.description.replace("\n", " ") + + for db_mod in db_modifiers: + if db_mod.static: + if db_mod.effect == effect: + static_roll = ItemModifierRoll(position=0) + return ItemModifierCreate( + modifierId=db_mod.modifierId, rolls=[static_roll] + ) + + elif (match := db_mod.regex.match(effect)) is not None: + rolls = self._process_dynamic_modifier(db_mod, match) + + return ItemModifierCreate(modifierId=db_mod.modifierId, rolls=rolls) + + return None + + def extract_modifiers(self, item: PoeItem) -> list[ItemModifierCreate] | None: + db_modifiers = self.modifiers[item.name] + + extracted_modifiers = list[ItemModifierCreate]() + if item.name == "The Adorned" and item.explicit_mods is None: + # rare case where the modifier dissapears completely from the item when it rolls 0 + # This can happen with other items, but they will still have some other mods left over + item.explicit_mods = [ + ItemMod( + description=r"0% increased Effect of Jewel Socket Passive Skills containing Corrupted Magic Jewels" + ) ] - - return "matched" + ":-:".join(rolls) - - dynamic_modifier_df = self.dynamic_modifier_df - - # The process must be broken down into a for-loop as the replacement is unique - - dynamic_w_rolls_df = dynamic_df.copy() - for effect, regex in dynamic_modifier_df[["effect", "regex"]].itertuples( - index=False - ): - matched_modifiers = dynamic_df["modifier"].str.replace( - regex, extract_rolls, regex=True, case=False - ) - matched_modifiers_mask = matched_modifiers.str.contains("matched", na=False) - - dynamic_w_rolls_df.loc[matched_modifiers_mask, "effect"] = effect - dynamic_w_rolls_df.loc[ - matched_modifiers_mask, "roll" - ] = matched_modifiers.loc[matched_modifiers_mask] - - dynamic_df.loc[matched_modifiers_mask, "modifier"] = pd.NA - - dynamic_w_rolls_df.loc[:, "roll"] = ( - dynamic_w_rolls_df["roll"].str.removeprefix("matched").str.split(":-:") - ) - del dynamic_df - dynamic_df = dynamic_w_rolls_df - - # If there are rows in the dataframe which contain empty lists, something has failed - failed_df = dynamic_df.loc[dynamic_df["roll"].isna()] - if not failed_df.empty: - logger.critical( - "Failed to add rolls to listed modifiers, this likely means" - " the modifier is legacy or there was a new expansion." - ) - logger.critical( - f"These items have missing modifiers: {failed_df['name'].unique().tolist()}" - ) - logger.critical( - f"These modifiers were not present in the database: {failed_df['effect'].unique().tolist()}" - ) - dynamic_df = dynamic_df.loc[~dynamic_df["roll"].isna()] - - return dynamic_df - - def _process_dynamic( - self, df: pd.DataFrame, static_modifers_mask: pd.Series - ) -> pd.DataFrame: - """ - A much more expensive operation - - Uses the regex column to match incoming modifiers to modifiers in the db. - """ - dynamic_modifier_df = self.dynamic_modifier_df - dynamic_df = df.loc[~static_modifers_mask] # Everything not static is dynamic - if dynamic_df.empty: - return pd.DataFrame( - columns=dynamic_df.columns.append(dynamic_modifier_df.columns) - ) - - dynamic_df.loc[:, "effect"] = dynamic_df.loc[:, "modifier"] - dynamic_df = self._get_rolls(dynamic_df.copy()) - - # Creates a column for position, which contains a list of numerical strings - dynamic_df.loc[:, "position"] = dynamic_df.loc[:, "roll"].apply( - lambda x: [str(i) for i in range(len(x))] - ) - - # Each row describes one roll - dynamic_df = dynamic_df.explode(["roll", "position"]) - - merged_dynamic_df = dynamic_df.merge( - dynamic_modifier_df, on=["effect", "position"], how="left" - ) - - # If all of these fields are still NA, it means that modifier was not matched with a modifier in our DB - failed_df = merged_dynamic_df.loc[merged_dynamic_df["roll"].isna()] - if not failed_df.empty: - logger.exception( - "Some modifiers did not find their counterpart in the database." - " This likely means the modifier is new or has been reworded.\n" - f"{failed_df[['effect', 'roll']].to_string()}" - ) - merged_dynamic_df = merged_dynamic_df.loc[~merged_dynamic_df["roll"].isna()] - - def convert_text_roll_to_index(row: pd.DataFrame) -> int: - text_rolls: str = row["textRolls"] - if text_rolls != "None": - text_rolls = text_rolls.lower().split("|") - roll = text_rolls.index(row["roll"].lower()) + elif item.explicit_mods is None: + logger.critical(f"An item was found with no explicit mods: {item}") + return None + + for mod in item.explicit_mods: + extracted_rolls = self._extract_rolls(mod, db_modifiers) + if extracted_rolls is None: + self.missing_modifiers[item.name].add( + mod.description.replace("\n", " ") + ) else: - roll = row["roll"] - - return roll + extracted_modifiers.append(extracted_rolls) - merged_dynamic_df.loc[:, "roll"] = merged_dynamic_df.apply( - convert_text_roll_to_index, axis=1 - ) # The `roll` column now contains a number + return extracted_modifiers - return merged_dynamic_df - - def add_rolls(self, df: pd.DataFrame) -> pd.DataFrame: - df = self._pre_processing(df.copy()) - - static_modifers_mask = df["modifier"].isin(self.static_modifier_df["effect"]) - - ready_static_df = self._process_static(df.copy(), static_modifers_mask) - ready_dynamic_df = self._process_dynamic(df.copy(), static_modifers_mask) - - processed_df = pd.concat( - (ready_static_df, ready_dynamic_df), axis=0, ignore_index=True - ) # static and dynamic item modifiers are combined into one dataframe again - - return processed_df + def log_missing_modifiers(self): + logger.critical( + "Failed to add rolls to listed modifiers, this likely means" + " the modifier are legacy or there was a new expansion." + f"Missing modifiers: {self.missing_modifiers}" + ) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py index 8465d070..ebaf86fa 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py @@ -1,138 +1,115 @@ -import pandas as pd +from collections.abc import Iterator + +from backend_api.app.core.schemas.currency import ( + Currency, + CurrencyPriceCreate, + CurrencyType, +) +from pydantic import TypeAdapter from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.currency import ( + ExchangeRatioItem, +) from data_retrieval_app.logs.logger import transform_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import get_data_safe, insert_data +from data_retrieval_app.utils import send_request_safe class TransformCurrencyAPIData: - def __init__(self) -> None: + def __init__(self, name_to_currency: dict[str, CurrencyType]) -> None: logger.debug("Initializing TransformCurrencyAPIData.") self.base_url = settings.BACKEND_BASE_URL - logger.debug(f"Url set to: {self.base_url}") + self.url = f"{self.base_url}/currency/price/" + logger.debug(f"Url set to: {self.url}") self.pom_api_headers = get_superuser_token_headers(self.base_url) logger.debug("Headers set to: " + str(self.pom_api_headers)) logger.debug("Initializing TransformCurrencyAPIData done.") - self.name_to_trade_name = self._get_name_to_trade_name_dict() + self.name_to_currency = name_to_currency - def _get_name_to_trade_name_dict(self) -> dict: - """ - Retrieves a map for "fancy" currency names, as used in the API, to their trade names, which we need. - """ - headers = { - "User-Agent": f"OAuth pathofmodifiers/0.1.0 (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" - } - response = get_data_safe( - "https://www.pathofexile.com/api/trade/data/static", - headers=headers, - logger=logger, - ) - - response_json = response.json() - result = response_json["result"] - currencies = {} - for category in result: - if category["id"] == "Currency": - for entry in category["entries"]: - name = entry["text"] - trade_name = entry["id"] - currencies[name] = trade_name - - return currencies - - def _transform_currency_table( - self, currency_df: pd.DataFrame, current_hours: dict[int, int] - ) -> pd.DataFrame: + def _transform( + self, + exchange_ratios: list[ExchangeRatioItem], + current_hours: dict[int, int], + ) -> dict[tuple[int, str], Currency]: """ Since a chaos orb is always worth one chaos orb, ninja does not include it in its price api. """ + currencies = dict[tuple[int, str], Currency]() + for ratio in exchange_ratios: + currency_type = self.name_to_currency.get(ratio.name) + if currency_type is None: + continue + + value = -1 + if ratio.chaos.chaosValue is None or ratio.chaos.chaosValue == 0: + value = ratio.divine.chaosValue + else: + value = ratio.chaos.chaosValue + + current_hour = current_hours[ratio.leagueId] + currency_id = currency_type.currencyId + trade_name = currency_type.tradeName + + currency = Currency( + currencyId=currency_id, + tradeName=trade_name, + name=ratio.name, + leagueId=ratio.leagueId, + createdHoursSinceLaunch=current_hour, + valueInChaos=value, + ) + + currencies[currency.leagueId, currency.tradeName] = currency + + for league_id, current_hour in current_hours.items(): + name = "Chaos Orb" + currency_type = self.name_to_currency[name] + currency_id = currency_type.currencyId + trade_name = currency_type.tradeName + chaos_currency = Currency( + currencyId=currency_id, + tradeName=trade_name, + name=name, + leagueId=league_id, + createdHoursSinceLaunch=current_hour, + valueInChaos=1, + ) + currencies[ + chaos_currency.leagueId, chaos_currency.tradeName + ] = chaos_currency - missing_chaos_value_mask = (currency_df["chaos.chaosValue"] == 0) | ( - currency_df["chaos.chaosValue"].isna() - ) - currency_df["chaos.chaosValue"] = currency_df["chaos.chaosValue"].where( - ~missing_chaos_value_mask, - currency_df["divine.chaosValue"], - ) - - chaos_dict = { - "name": ["Chaos Orb"], - "chaos.chaosValue": [1], - } - for league_id in currency_df["leagueId"].unique(): - chaos_dict["leagueId"] = [league_id] - chaos_df = pd.DataFrame.from_dict(chaos_dict) - currency_df = pd.concat((currency_df, chaos_df), ignore_index=True) - - currency_df["tradeName"] = currency_df["name"].map( - lambda name: self.name_to_trade_name.get(name, pd.NA) - ) - - currency_df["createdHoursSinceLaunch"] = currency_df["leagueId"].map( - current_hours - ) - return currency_df - - def _clean_currency_table(self, currency_df: pd.DataFrame) -> pd.DataFrame: - """ - Cleans the currency table of unnecessary columns. - """ - currency_df = currency_df.rename(columns={"chaos.chaosValue": "valueInChaos"}) - - currency_df = currency_df.drop( - currency_df.columns.difference( - ["tradeName", "valueInChaos", "createdHoursSinceLaunch", "leagueId"] - ), - axis=1, - ) - currency_df = currency_df.loc[~currency_df["tradeName"].isna()].reset_index( - drop=True - ) - return currency_df - - def _get_latest_currency_id_series(self, currency_df: pd.DataFrame) -> pd.Series: - response = get_data_safe( - f"{self.base_url}/currency/latest_currency_id/", - headers=self.pom_api_headers, - logger=logger, - ) - latest_currency_id = int(response.text) + return currencies - currency_id = pd.Series( - range(latest_currency_id - len(currency_df) + 1, latest_currency_id + 1), - dtype=int, + def _insert(self, currencies: Iterator[Currency]): + prices = list[CurrencyPriceCreate]() + for currency in currencies: + prices.append( + CurrencyPriceCreate( + currencyId=currency.currencyId, + leagueId=currency.leagueId, + createdHoursSinceLaunch=currency.createdHoursSinceLaunch, + valueInChaos=currency.valueInChaos, + ) + ) + + headers = {"accept": "application/json", "Content-Type": "application/json"} + headers.update(self.pom_api_headers) + send_request_safe( + "post", + self.url, + json=TypeAdapter(list[CurrencyPriceCreate]).dump_python(prices), + headers=headers, ) - return currency_id - def transform_into_tables( - self, currency_df: pd.DataFrame, current_hours: dict[int, int] - ) -> pd.DataFrame: - """ - Transforms the data into tables and transforms with help functions. - """ - logger.debug("Transforming data into tables.") - currency_df = self._transform_currency_table(currency_df, current_hours) - logger.debug("Successfully transformed data into tables.") - - logger.debug("Cleaning currency table data.") - currency_df = self._clean_currency_table(currency_df) - logger.debug("Successfully cleaned currency table data.") - - logger.debug("Inserting currency data into database.") - insert_data( - currency_df, - url=self.base_url, - table_name="currency", - logger=logger, - headers=self.pom_api_headers, - ) + def transform_and_insert( + self, exchange_ratios: list[ExchangeRatioItem], current_hours: dict[int, int] + ) -> dict[tuple[int, str], Currency]: + logger.debug("Transforming exchange ratios into currencies.") + currencies = self._transform(exchange_ratios, current_hours) + logger.debug("Inserting currency prices.") + self._insert(currencies.values()) logger.debug("Successfully inserted currency data into database.") - currency_id = self._get_latest_currency_id_series(currency_df) - logger.debug("Latest currency id found: " + str(currency_id)) - - currency_df = currency_df.assign(currencyId=currency_id) - logger.debug("Successfully transformed data into tables.") - return currency_df + return currencies diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py index 6f4700f9..2931c6b8 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py @@ -1,619 +1,357 @@ -from typing import Any - -import pandas as pd -from requests.exceptions import HTTPError +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import Literal, TypeVar + +from backend_api.app.core.schemas.currency import Currency +from backend_api.app.core.schemas.item import ( + ItemAvailabilityExpired, + ItemAvailabilityUpdated, + ItemBase, + ItemCreate, + ItemPrice, +) +from backend_api.app.core.schemas.item_base_type import ItemBaseType +from backend_api.app.core.schemas.league import League +from backend_api.app.core.schemas.modifier import GroupedModifier +from pydantic import BaseModel, TypeAdapter from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.external.poe import ( + PoeItem, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.cache import ( + CacheItemWithContext, +) +from data_retrieval_app.external_data_retrieval.data_retrieval.schemas.internal.categorized import ( + OrganizedItems, + OrganizedItemsByCategory, +) +from data_retrieval_app.external_data_retrieval.detectors.base import ( + CategoryDetectorBase, +) +from data_retrieval_app.external_data_retrieval.detectors.unique_detector import ( + # UnidentifiedUniqueDetector, + UniqueDetector, +) from data_retrieval_app.external_data_retrieval.transforming_data.roll_processor import ( RollProcessor, ) from data_retrieval_app.external_data_retrieval.utils import sync_timing_tracker from data_retrieval_app.logs.logger import transform_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers -from data_retrieval_app.utils import get_data_safe, insert_data +from data_retrieval_app.utils import send_request_safe -pd.options.mode.chained_assignment = None # default="warn" +class PoEAPIDataTransformer: + url = settings.BACKEND_BASE_URL + pom_auth_headers = get_superuser_token_headers(url) -class PoEAPIDataTransformerBase: def __init__( self, - leagues: list[dict[str, Any]], + leagues: list[League], ) -> None: logger.debug("Initializing PoEAPIDataTransformer") - self.base_url = settings.BACKEND_BASE_URL - self.pom_auth_headers = get_superuser_token_headers(self.base_url) - self.roll_processor = RollProcessor() + self.pom_auth_headers = get_superuser_token_headers(self.url) - self.league_to_id = {league["name"]: league["leagueId"] for league in leagues} + modifiers = self._get_modifiers() + item_base_types = self._get_item_base_types() - logger.debug("Initializing PoEAPIDataTransformer done.") - - def _create_item_table(self, df: pd.DataFrame) -> pd.DataFrame: - """ - Creates the basis of the `item` table. - """ - self.item_columns = [ - "itemId", - "id", - "name", - "league", - "baseType", - "typeLine", - "ilvl", - "rarity", - "identified", - "note", - "corrupted", - "delve", - "fractured", - "synthesised", - "replica", - "influences.shaper", - "influences.elder", - "influences.crusader", - "influences.hunter", - "influences.redeemer", - "influences.warlord", - "extended.prefixes", - "extended.suffixes", - "searing", - "tangled", - "foilVariation", - "isRelic", - "stash", - ] - item_df = df.loc[ - :, [column for column in self.item_columns if column in df.columns] - ] # Can't guarantee all columns are present + self.roll_processor = RollProcessor(modifiers) - return item_df - - def _find_not_too_highly_priced_item_mask( - self, - currency_df: pd.DataFrame, - item_currency_merged_df: pd.DataFrame, - ) -> pd.Series: - """ - Some items are too highly priced to be legitimate. A boundary of the price equivelant to - >= 10 mirrors are seen to be too high priced. - - 1. Find currency type and amount equiveland in chaos - """ - - mirror_row = currency_df.loc[currency_df["tradeName"] == "mirror"] - mirror_value = mirror_row["valueInChaos"].get(0, 100_000) - - currency_too_high_mask = ( - item_currency_merged_df["valueInChaos"] - * item_currency_merged_df["currencyAmount"].astype("float") - > mirror_value * 10 - ) + self.league_to_id = {league.name: league.leagueId for league in leagues} - return ~currency_too_high_mask + logger.debug("Initializing PoEAPIDataTransformer done.") - @sync_timing_tracker - def _transform_item_table( - self, - item_df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - The `item` table requires a foreign key to the `currency` table. - Everything related to the price of the item is stored in the `note` - attribute. - - There are two types of listings in PoE, exact price and asking price which are - represented by `price` and `b/o` respectively. - """ - - def transform_base_types(element): - return item_base_types[element] - - def get_currency_amount(element): - if len(element) == 3: - return element[1] - return pd.NA - - def get_currency_type(element): - if len(element) == 3: - return element[2] - return "" - - def transform_influences(row: pd.DataFrame, influence_columns: list[str]): - if not row[influence_columns].any(): - return pd.NA - else: - influence_dict = {} - for influence_column in influence_columns: - if row[influence_column]: - influence_dict[ - influence_column.replace("influences.", "") - ] = True - return influence_dict - - item_df["leagueId"] = item_df["league"].map(self.league_to_id) - item_df["createdHoursSinceLaunch"] = item_df["leagueId"].map(current_hours) - - base_type_series = item_df["baseType"] - item_df["itemBaseTypeId"] = base_type_series.apply(transform_base_types) - - influence_columns = [ - column for column in item_df.columns if "influences" in column + self.transformers: list[TransformerBase] = [ + UniqueTransformer(self.league_to_id, item_base_types, self.roll_processor), + # UnidentifiedUniqueTransformer(self.league_to_id, item_base_types, self.roll_processor), ] - item_df["influences"] = item_df.apply( - lambda row: transform_influences(row, influence_columns), axis=1 + + def _get_modifiers(self) -> dict[str, GroupedModifier]: + response = send_request_safe( + "get", + f"{self.url}/modifier/grouped/", + headers=self.pom_auth_headers, + logger=logger, ) + modifiers = TypeAdapter(list[GroupedModifier]).validate_python(response.json()) - rename_extended_map = {} - if "extended.prefixes" in item_df.columns: - rename_extended_map["extended.prefixes"] = "prefixes" - if "extended.suffixes" in item_df.columns: - rename_extended_map["extended.suffixes"] = "suffixes" + name_to_modifiers = defaultdict[str, list[GroupedModifier]](list) + for modifier in modifiers: + uniques = modifier.relatedUniques.split("|") + for unique in uniques: + name_to_modifiers[unique].append(modifier) - rename_map = {**rename_extended_map, "id": "gameItemId"} - item_df = item_df.rename(columns=rename_map) + return dict(name_to_modifiers) - note_series = None - if "note" in item_df.columns: - note_series = item_df["stash"].str.split(" ") + def _get_item_base_types(self) -> dict[str, int]: + response = send_request_safe( + "get", + f"{self.url}/itemBaseType/", + headers=self.pom_auth_headers, + logger=logger, + ) - stash_series = None - if "stash" in item_df.columns: - stash_series = item_df["stash"].str.split(" ") + base_types = TypeAdapter(list[ItemBaseType]).validate_python(response.json()) + base_type_ids = { + base_type.baseType: base_type.itemBaseTypeId for base_type in base_types + } + return base_type_ids - if note_series is not None: - currency_series = note_series - if stash_series is not None: - currency_series = currency_series.where( - item_df["note"].str.contains("~"), stash_series - ) - else: - currency_series = stash_series + def set_current_hours(self, current_hours: dict[int, int]): + for transformer in self.transformers: + transformer.set_current_hours(current_hours) - item_df["currencyAmount"] = currency_series.apply(get_currency_amount) - item_df["currencyType"] = currency_series.apply(get_currency_type) + def set_currencies(self, currencies: dict[tuple[int, str], Currency]): + for transformer in self.transformers: + transformer.set_currencies(currencies) - invalid_amount_mask = ~item_df["currencyAmount"].str.match( - r"^(([0-9]*[.])?[0-9]+)$", na=False - ) - item_df.loc[invalid_amount_mask, "currencyAmount"] = pd.NA - item_df.loc[invalid_amount_mask, "currencyType"] = "" - - item_df = item_df.merge( - currency_df, - how="left", - left_on=["currencyType", "leagueId"], - right_on=["tradeName", "leagueId"], - suffixes=(None, "_y"), - ) + @sync_timing_tracker + def transform_and_insert( + self, + organized_items: OrganizedItemsByCategory, + ): + for transformer in self.transformers: + transformer.transform_and_insert( + getattr(organized_items, transformer.identifier), + ) - price_found_mask = ~item_df["tradeName"].isna() + def end_of_hour_cleanup(self): + self.roll_processor.log_missing_modifiers() + for transformer in self.transformers: + transformer.end_of_hour_cleanup() - self.price_found_mask = price_found_mask - item_df = item_df.loc[self.price_found_mask] +CreateSchemaType = TypeVar("CreateSchemaType", bound=BaseModel) +UpdateSchemaType = TypeVar("UpdateSchemaType", bound=BaseModel) - not_too_high_priced_item_mask = self._find_not_too_highly_priced_item_mask( - currency_df, item_df - ) - self.items_not_too_high_priced_mask = not_too_high_priced_item_mask - - item_df = item_df.loc[self.items_not_too_high_priced_mask] - - return item_df - - @property - def item_table_columns_to_not_drop(self) -> set[str]: - try: - dont_drop_columns = self._item_table_columns_to_not_drop - except AttributeError: - dont_drop_columns = { - "gameItemId", - "name", - "leagueId", - "itemBaseTypeId", - "ilvl", - "rarity", - "identified", - "currencyAmount", - "currencyId", - "corrupted", - "delve", - "fractured", - "synthesised", - "replica", - "influences", - "searing", - "tangled", - "prefixes", - "suffixes", - "foilVariation", - "createdHoursSinceLaunch", - } - self._item_table_columns_to_not_drop = dont_drop_columns - return dont_drop_columns - - @item_table_columns_to_not_drop.setter - def item_table_columns_to_not_drop(self, columns_to_not_drop: set[str]) -> set[str]: - self._item_table_columns_to_not_drop = columns_to_not_drop - - def _update_item_table_columns_to_not_drop(self, *, dont_drop: set[str]) -> None: - columns_to_not_drop = self.item_table_columns_to_not_drop - - self.item_table_columns_to_not_drop = columns_to_not_drop | dont_drop - - def _clean_item_table(self, item_df: pd.DataFrame) -> pd.DataFrame: - """ - Gets rid of unnecessay information, so that only fields needed for the DB remains. - """ - - dont_drop_columns = self.item_table_columns_to_not_drop - - item_df.drop( - item_df.columns.difference(dont_drop_columns), - axis=1, - inplace=True, - errors="ignore", - ) - return item_df +class TransformerBase[RemoveSchemaType, UpdateSchemaType, CreateSchemaType](ABC): + detector: CategoryDetectorBase + identifier: str - def _get_latest_item_id_series(self, item_df: pd.DataFrame) -> pd.Series: - response = get_data_safe( - f"{self.base_url}/item/latest_item_id/", - headers=self.pom_auth_headers, - logger=logger, - ) - latest_item_id = int(response.text) + url = settings.BACKEND_BASE_URL + sub_path: str + url: str - item_id = pd.Series( - range(latest_item_id - len(item_df) + 1, latest_item_id + 1), dtype=int - ) + remove_subpath: str | None = None + update_subpath: str | None = None + insert_subpath: str | None = None - return item_id + pom_auth_headers = get_superuser_token_headers(url) - def _process_item_table( + def __init__( self, - df: pd.DataFrame, - currency_df: pd.DataFrame, + league_to_id: dict[str, int], item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> pd.Series: - """ - Needs to return item ids, as it is used to connect the item modifiers - """ - item_df = self._create_item_table(df) - item_df = self._transform_item_table( - item_df, currency_df, item_base_types, current_hours=current_hours - ) - item_df = self._clean_item_table(item_df) - insert_data( - item_df, - url=self.base_url, - table_name="item", - logger=logger, - headers=self.pom_auth_headers, - ) - item_id = self._get_latest_item_id_series(item_df) - logger.debug("Latest item id found: " + str(item_id)) - return item_id + roll_processor: RollProcessor, + ): + self.identifier = self.detector.identifier + self.url = f"{self.url}/{self.sub_path}/" - @sync_timing_tracker - def _transform_unidentified_item_table( - self, - item_df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - For convenience, all unid items are stored in divine prices - """ - item_df = self._transform_item_table( - item_df, currency_df, item_base_types, current_hours=current_hours - ) + self.type_adapters: dict[Literal["remove", "update", "insert"], TypeAdapter] = { + "remove": TypeAdapter(list[RemoveSchemaType]), + "update": TypeAdapter(list[UpdateSchemaType]), + "insert": TypeAdapter(list[CreateSchemaType]), + } - item_df["chaos_value"] = ( - item_df["currencyAmount"].astype(float) * item_df["valueInChaos"] + self.league_to_id = league_to_id + self.item_base_types = item_base_types + self.roll_processor = roll_processor + + def set_current_hours(self, current_hours: dict[int, int]): + self.current_hours = current_hours + + def set_currencies(self, currencies: dict[tuple[int, str], Currency]): + self.currencies = currencies + self.league_to_mirror = dict[int, Currency]() + for league_id in self.league_to_id.values(): + mirror = currencies[(league_id, "mirror")] + self.league_to_mirror[league_id] = mirror + + @abstractmethod + def _transform_remove( + self, removed_items: list[CacheItemWithContext], current_hours: dict[int, int] + ) -> list[RemoveSchemaType]: + """Transforms items into objects for removing""" + + def _remove(self, removed_items: list[RemoveSchemaType]): + remove_url = self.url + if self.remove_subpath is not None: + remove_url += f"{self.remove_subpath}/" + + send_request_safe( + "patch", + remove_url, + json=self.type_adapters["remove"].dump_python(removed_items), + headers=self.pom_auth_headers, ) - for league in item_df["leagueId"].unique(): - divine_row = currency_df.loc[ - (currency_df["leagueId"] == league) - & (currency_df["tradeName"] == "divine") - ].iloc[0] + def _extract_price(self, item: PoeItem) -> ItemPrice | None: + league_id = self.league_to_id[item.league] - divine_id = divine_row["currencyId"] - divine_value = divine_row["valueInChaos"] + _, currency_amount_str, currency_type_str = item.note.split() - item_league_mask = item_df["leagueId"] == league - item_df.loc[item_league_mask, "currencyId"] = divine_id - item_df.loc[item_league_mask, "currencyAmount"] = ( - item_df.loc[item_league_mask, "chaos_value"] / divine_value - ) - return item_df - - def _clean_unidentified_item_table(self, item_df: pd.DataFrame) -> pd.DataFrame: - """ - Gets rid of unnecessay information, so that only fields needed for the DB remains. - - Only selects unidentified items and always drops suffixes and prefixes - """ - - dont_drop_columns = { - "name", - "itemBaseTypeId", - "createdHoursSinceLaunch", - "leagueId", - "currencyId", - "ilvl", - "currencyAmount", - "identified", - "rarity", - } - - item_df.drop( - item_df.columns.difference(dont_drop_columns), - axis=1, - inplace=True, - errors="ignore", - ) + currency_type = self.currencies.get((league_id, currency_type_str)) + if currency_type is None: + return None - unidentified_item_df = item_df.loc[~item_df["identified"]] + currency_amount = float(currency_amount_str) - return unidentified_item_df + chaos_price = currency_type.valueInChaos * currency_amount + if self._exceeds_price_threshold(league_id, chaos_price): + return None - def _aggregate_unidentified_item_table( - self, - ): - """ - Should run once at the end of every hour. For safety, it also aggregates all previous - hours, in case of previous unfortunate errors. - """ - response = get_data_safe( - f"{self.base_url}/unidentifiedItem/non_aggregated/", - headers=self.pom_auth_headers, - logger=logger, + return ItemPrice( + currencyId=currency_type.currencyId, currencyAmount=currency_amount ) - unid_df = pd.DataFrame(response.json()) - if unid_df.empty: - logger.info("Found no unidentified items to aggregate") - return - - group_cols = [ - "leagueId", - "name", - "itemBaseTypeId", - "createdHoursSinceLaunch", - "ilvl", - "identified", - "currencyId", - "rarity", - ] - g = unid_df.groupby(group_cols) + def _exceeds_price_threshold(self, league_id: int, chaos_price: float) -> bool: + mirror = self.league_to_mirror[league_id] + price_in_mirrors = chaos_price / mirror.valueInChaos - unid_df["calc_avg"] = g["currencyAmount"].transform("mean") - unid_df["calc_std"] = g["currencyAmount"].transform("std") - unid_df["calc_count"] = g["itemId"].transform("count") + return price_in_mirrors > settings.MAX_MIRROR_PRICE - filtered_df = unid_df[ - unid_df["currencyAmount"].between( - unid_df["calc_avg"] - 1.97 * unid_df["calc_std"], - unid_df["calc_avg"] + 1.97 * unid_df["calc_std"], - ) - | unid_df["calc_std"].isna() - ] + @abstractmethod + def _transform_update(self, changed_items: list[PoeItem]) -> list[UpdateSchemaType]: + """Transforms items into objects for updating""" - result_df = filtered_df.groupby(group_cols, as_index=False).agg( - currencyAmount=("currencyAmount", "mean"), - nItems=("calc_count", "first"), - ) - result_df["aggregated"] = True - logger.info("Pushing aggregated unidentified items") - insert_data( - result_df, - url=self.base_url, - table_name="unidentifiedItem/add_aggregated", - logger=logger, + def _update(self, changed_items: list[UpdateSchemaType]): + update_url = self.url + if self.update_subpath is not None: + update_url += f"{self.update_subpath}/" + + send_request_safe( + "put", + update_url, + json=self.type_adapters["update"].dump_python( + changed_items, exclude_none=True + ), headers=self.pom_auth_headers, ) - def _process_unidentified_item_table( - self, - df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> None: - item_df = self._create_item_table(df) - item_df = self._transform_unidentified_item_table( - item_df, currency_df, item_base_types, current_hours=current_hours - ) - item_df = self._clean_unidentified_item_table(item_df) - insert_data( - item_df, - url=self.base_url, - table_name="unidentifiedItem", - logger=logger, + @abstractmethod + def _transform(self, new_items: list[PoeItem]) -> CreateSchemaType: + """Transforms items, using currencies info, into a create schema for the relevant table""" + + def _insert(self, transformed: list[CreateSchemaType]): + insert_url = self.url + if self.insert_subpath is not None: + insert_url += f"{self.insert_subpath}/" + + send_request_safe( + "post", + insert_url, + json=self.type_adapters["insert"].dump_python( + transformed, exclude_none=True + ), headers=self.pom_auth_headers, ) - def _create_item_modifier_table( - self, df: pd.DataFrame, *, item_id: pd.Series - ) -> pd.DataFrame: - """ - The `item_modifier` table heavily relies on what type of item the modifiers - belong to. - """ - raise NotImplementedError("Only available in child classes") + def transform_and_insert(self, items: OrganizedItems): + if items.removed_items: + transformed_remove = self._transform_remove(items.removed_items) + self._remove(transformed_remove) - def _transform_item_modifier_table( - self, - item_modifier_df: pd.DataFrame, - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - The `item_modifier` table heavily relies on what type of item the modifiers - belong to. - """ - raise NotImplementedError("Only available in child classes") - - def _clean_item_modifier_table( - self, item_modifier_df: pd.DataFrame - ) -> pd.DataFrame: - """ - The `item_modifier` table heavily relies on what type of item the modifiers - belong to. - - Gets rid of unnecessay information, so that only fields needed for the DB remains. - """ - raise NotImplementedError("Only available in child classes") - - def _process_item_modifier_table( - self, - df: pd.DataFrame, - item_id: pd.Series, - current_hours: dict[int, int], - ) -> None: - item_modifier_df = self._create_item_modifier_table( - df, item_id=item_id, current_hours=current_hours - ) - item_modifier_df = self._transform_item_modifier_table(item_modifier_df) - item_modifier_df = self._clean_item_modifier_table(item_modifier_df) - insert_data( - item_modifier_df, - url=self.base_url, - table_name="itemModifier", - logger=logger, - headers=self.pom_auth_headers, - ) + if items.changed_items: + transformed_update = self._transform_update(items.changed_items) + self._update(transformed_update) - def transform_into_tables( - self, - df: pd.DataFrame, - modifier_df: pd.DataFrame, - currency_df: pd.DataFrame, - item_base_types: dict[str, int], - current_hours: dict[int, int], - ) -> None: - self.roll_processor.add_modifier_df(modifier_df) - try: - logger.debug("Transforming data into tables.") - logger.debug("Processing data tables.") - item_id = self._process_item_table( - df.copy(deep=True), - currency_df=currency_df, - item_base_types=item_base_types, - current_hours=current_hours, - ) - self._process_unidentified_item_table( - df.copy(deep=True), - currency_df=currency_df, - item_base_types=item_base_types, - current_hours=current_hours, + if items.new_items: + transformed = self._transform(items.new_items) + self._insert(transformed) + + @abstractmethod + def end_of_hour_cleanup(self): + """Does necessary cleanup at the end of the hour (eg. unidentified items)""" + + +class UniqueTransformer( + TransformerBase[ItemAvailabilityExpired, ItemAvailabilityUpdated, ItemCreate] +): + detector = UniqueDetector() + sub_path = "item" + + remove_subpath = "availability" + update_subpath = "availability" + + def _transform_remove( + self, removed_items: list[CacheItemWithContext] + ) -> list[ItemAvailabilityExpired]: + transformed = list[ItemAvailabilityExpired]() + for item in removed_items: + league_id = self.league_to_id[item.league] + + transformed.append( + ItemAvailabilityExpired( + gameItemId=item.id, + leagueId=league_id, + validTo=self.current_hours[league_id] + 1, + ) ) - self._process_item_modifier_table( - df.copy(deep=True), - item_id=item_id, - current_hours=current_hours, + + return transformed + + def _transform_update( + self, changed_items: list[PoeItem] + ) -> list[ItemAvailabilityUpdated]: + patches = list[ItemAvailabilityUpdated]() + for item in changed_items: + game_item_id = item.id + league_id = self.league_to_id[item.league] + + price = self._extract_price(item) + if price is None: + continue + + patches.append( + ItemAvailabilityUpdated( + gameItemId=game_item_id, + leagueId=league_id, + price=price, + validFrom=self.current_hours[league_id], + ) ) - logger.debug("Successfully transformed data into tables.") - except HTTPError as e: - logger.exception(f"Something went wrong:\n{repr(e)}") - raise e + return patches - def end_of_hour_cleanup(self): - self._aggregate_unidentified_item_table() + def _transform(self, new_items: list[PoeItem]) -> ItemCreate: + transformed = list[ItemCreate]() + for item in new_items: + game_item_id = item.id + league_id = self.league_to_id[item.league] + price = self._extract_price(item) + if price is None: + continue -class UniquePoEAPIDataTransformer(PoEAPIDataTransformerBase): - @sync_timing_tracker - def _create_item_modifier_table( - self, - df: pd.DataFrame, - *, - item_id: pd.Series, - current_hours: dict[int, int], - ) -> pd.DataFrame: - """ - A similiar process to creating the item table, only this time the - relevant column contains a list and not a JSON-object - """ - item_modifier_columns = ["name", "explicitMods", "league"] - item_modifier_df = df.loc[ - self.price_found_mask, - item_modifier_columns, - ] + modifiers = self.roll_processor.extract_modifiers(item) + if modifiers is None: + continue - item_modifier_df = item_modifier_df.loc[ - self.items_not_too_high_priced_mask - ].reset_index() + item_base_type_id = self.item_base_types[item.base_type] - item_modifier_df["itemId"] = item_id - item_modifier_df["leagueId"] = item_modifier_df["league"].map(self.league_to_id) - item_modifier_df["createdHoursSinceLaunch"] = item_modifier_df["leagueId"].map( - current_hours - ) - item_modifier_df = item_modifier_df.explode("explicitMods", ignore_index=True) + first_observed = self.current_hours[league_id] - item_modifier_df.rename({"explicitMods": "modifier"}, axis=1, inplace=True) + item_base = ItemBase( + game_item_id=game_item_id, + league_id=league_id, + item_base_type_id=item_base_type_id, + first_observed=first_observed, + **item.model_dump(), + ) - return item_modifier_df + transformed.append( + ItemCreate(item=item_base, price=price, modifiers=modifiers) + ) + + return transformed + + def end_of_hour_cleanup(self): + pass - @sync_timing_tracker - def _transform_item_modifier_table( - self, - item_modifier_df: pd.DataFrame, - ) -> pd.DataFrame: - item_modifier_df = self.roll_processor.add_rolls(df=item_modifier_df) - - return item_modifier_df - - @property - def item_modifier_table_columns_to_not_drop(self) -> set[str]: - try: - dont_drop_columns = self._item_modifier_table_columns_to_not_drop - except AttributeError: - dont_drop_columns = { - "itemId", - "modifierId", - "position", - "roll", - "createdHoursSinceLaunch", - } - self._item_modifier_table_columns_to_not_drop = dont_drop_columns - return dont_drop_columns - - @item_modifier_table_columns_to_not_drop.setter - def item_modifier_table_columns_to_not_drop( - self, columns_to_not_drop: set[str] - ) -> set[str]: - self._item_modifier_table_columns_to_not_drop = columns_to_not_drop - - def _update_item_modifier_table_columns_to_not_drop( - self, *, dont_drop: set[str] - ) -> None: - columns_to_not_drop = self.item_modifier_table_columns_to_not_drop - self.item_modifier_table_columns_to_not_drop = columns_to_not_drop | dont_drop - - def _clean_item_modifier_table( - self, item_modifier_df: pd.DataFrame - ) -> pd.DataFrame: - """ - Gets rid of unnecessay information, so that only fields needed for the DB remains. - """ - dont_drop_columns = self.item_modifier_table_columns_to_not_drop - - item_modifier_df.drop( - item_modifier_df.columns.difference(dont_drop_columns), - axis=1, - inplace=True, - ) - return item_modifier_df +# class UnidentifiedUniqueTransformer(TransformerBase): +# detector = UnidentifiedUniqueDetector() +# sub_path = "unidentified_item" diff --git a/src/backend_data_retrieval/data_retrieval_app/logs/logger.py b/src/backend_data_retrieval/data_retrieval_app/logs/logger.py index fd5b27f3..956413ca 100644 --- a/src/backend_data_retrieval/data_retrieval_app/logs/logger.py +++ b/src/backend_data_retrieval/data_retrieval_app/logs/logger.py @@ -24,9 +24,12 @@ def setup_logging() -> None: timing_logger = external_data_retrieval_logger.getChild("timing") +data_retrieval_logger = external_data_retrieval_logger.getChild("dataret") + +filter_logger = external_data_retrieval_logger.getChild("filter") + transform_logger = external_data_retrieval_logger.getChild("transform") -data_retrieval_logger = external_data_retrieval_logger.getChild("dataret") data_deposit_logger = main_logger.getChild("datadepo") diff --git a/src/backend_data_retrieval/data_retrieval_app/utils.py b/src/backend_data_retrieval/data_retrieval_app/utils.py index 0e0d6f89..804ee8dc 100644 --- a/src/backend_data_retrieval/data_retrieval_app/utils.py +++ b/src/backend_data_retrieval/data_retrieval_app/utils.py @@ -1,10 +1,11 @@ import logging from collections.abc import Generator from datetime import UTC, datetime -from typing import Any +from typing import Any, Literal import pandas as pd import requests +from backend_api.app.core.schemas.league import League from pydantic import HttpUrl from data_retrieval_app.logs.logger import main_logger as logger @@ -53,23 +54,21 @@ def df_to_JSON( ) -def find_hours_since_launch(leagues_df: list[dict]) -> dict[int, int]: +def find_hours_since_launch(leagues: list[League]) -> dict[int, int]: """ Finds the number of hours since launch for each of the leagues in the given dataframe """ current_time = datetime.now(UTC) - hours_since_launch_dict = {} - for league in leagues_df: - league_launch_time = datetime.fromisoformat(league["validFrom"]) - - time_since_launch = current_time - league_launch_time + hours_since_launch_dict = dict[int, int]() + for league in leagues: + time_since_launch = current_time - league.validFrom days_since_launch, seconds_since_launch = ( time_since_launch.days, time_since_launch.seconds, ) hours_since_launch = days_since_launch * 24 + seconds_since_launch // 3600 - hours_since_launch_dict[league["leagueId"]] = hours_since_launch + hours_since_launch_dict[league.leagueId] = hours_since_launch return hours_since_launch_dict @@ -142,7 +141,42 @@ def get_data_safe( except Exception as e: if logger is not None: logger.error( - f"The following error occurred while making request a request to {url}: {e}" + f"The following error occurred while making request a get request to {url}: {e}" + ) + raise e + + return response + + +def send_request_safe( + method: Literal["get", "put", "patch", "post", "delete"], + url: str, + *args, + logger: logging.Logger = None, + **kwargs, +) -> requests.Response: + method_func = None + if method == "get": + method_func = requests.get + elif method == "put": + method_func = requests.put + elif method == "patch": + method_func = requests.patch + elif method == "post": + method_func = requests.post + elif method == "delete": + method_func = requests.delete + else: + raise ValueError( + f"Invalid method ({method}), must be one of: get, put, patch, post, delete" + ) + try: + response = method_func(url, *args, **kwargs) + response.raise_for_status() + except Exception as e: + if logger is not None: + logger.error( + f"The following error occurred while making request a {method} request to {url}: {e}" ) raise e diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml index 02a72fae..b940236f 100644 --- a/src/docker-compose.override.yml +++ b/src/docker-compose.override.yml @@ -64,7 +64,7 @@ services: watch: - path: ./backend_api action: sync - target: /app + target: /app/backend_api ignore: - .venv - uv.lock @@ -92,6 +92,8 @@ services: - "8889:8889" build: context: ./backend_data_retrieval + additional_contexts: + backend_api: ./backend_api args: INSTALL_DEV: ${INSTALL_DEV-true} command: /data_retrieval_app/prestart.sh @@ -105,6 +107,9 @@ services: - uv.lock - path: ./backend_data_retrieval/pyproject.toml action: rebuild + - path: ./backend_api/app/core/schemas + action: sync+restart + target: /data_retrieval_app/backend_api/app/core/schemas frontend: restart: "no" diff --git a/src/docker-compose.yml b/src/docker-compose.yml index ee76fc8c..8c8851f5 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -41,7 +41,6 @@ services: - node.labels.${STACK_NAME?Variable not set}.pom-cache-data == true environment: REDIS_PASSWORD: ${REDIS_PASSWORD?Variable not set} - REDIS_CACHE: ${REDIS_CACHE?Variable not set} volumes: - pom-cache-data:/data healthcheck: @@ -121,6 +120,8 @@ services: - .env build: context: ./backend_data_retrieval + additional_contexts: + backend_api: ./backend_api args: INSTALL_DEV: ${INSTALL_DEV-false} @@ -179,7 +180,7 @@ services: volumes: pom-db-data: - name: "pom-db-default-data" + name: "830-test-pom-db-default-data" pom-cache-data: name: "pom-cache-data" pgadmin-data: