expanded api to include bdx feeds, ga4 properties, ga4 streams, go cl…

…oud properties
joeygrable94 · Apr 29, 2024 · efda0c5 · efda0c5
1 parent 18c6637
commit efda0c5
Show file tree

Hide file tree

Showing 72 changed files with 4,905 additions and 520 deletions.
diff --git a/alembic/versions/67ab88f410ce_added_website_id_to_ga4_stream_removed_.py b/alembic/versions/67ab88f410ce_added_website_id_to_ga4_stream_removed_.py
@@ -0,0 +1,54 @@
+"""added website id to ga4 stream, removed from ga4 property
+
+Revision ID: 67ab88f410ce
+Revises: cce84fb59468
+Create Date: 2024-04-27 16:11:39.934879
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+import sqlalchemy_utils  # type: ignore
+from app.db.custom_types import LongText
+
+# revision identifiers, used by Alembic.
+revision = '67ab88f410ce'
+down_revision = 'cce84fb59468'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_unique_constraint(None, 'go_a4', ['property_id'])
+    op.drop_constraint('go_a4_ibfk_2', 'go_a4', type_='foreignkey')
+    op.drop_column('go_a4', 'website_id')
+    op.add_column('go_a4_stream', sa.Column('website_id', sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False))
+    op.create_foreign_key(None, 'go_a4_stream', 'website', ['website_id'], ['id'])
+    op.alter_column('website_keywordcorpus', 'corpus',
+               existing_type=mysql.LONGBLOB(),
+               type_=LongText(),
+               existing_nullable=False)
+    op.alter_column('website_keywordcorpus', 'rawtext',
+               existing_type=mysql.LONGBLOB(),
+               type_=LongText(),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('website_keywordcorpus', 'rawtext',
+               existing_type=LongText(),
+               type_=mysql.LONGBLOB(),
+               existing_nullable=False)
+    op.alter_column('website_keywordcorpus', 'corpus',
+               existing_type=LongText(),
+               type_=mysql.LONGBLOB(),
+               existing_nullable=False)
+    op.drop_constraint(None, 'go_a4_stream', type_='foreignkey')  # type: ignore
+    op.drop_column('go_a4_stream', 'website_id')
+    op.add_column('go_a4', sa.Column('website_id', mysql.CHAR(length=32), nullable=False))
+    op.create_foreign_key('go_a4_ibfk_2', 'go_a4', 'website', ['website_id'], ['id'])
+    op.drop_constraint(None, 'go_a4', type_='unique')  # type: ignore
+    # ### end Alembic commands ###
diff --git a/alembic/versions/c934ef207888_revised_bdx_feed_table.py b/alembic/versions/c934ef207888_revised_bdx_feed_table.py
@@ -0,0 +1,45 @@
+"""revised bdx feed table
+
+Revision ID: c934ef207888
+Revises: 67ab88f410ce
+Create Date: 2024-04-28 14:04:00.705835
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+from app.db.custom_types import LongText
+
+# revision identifiers, used by Alembic.
+revision = 'c934ef207888'
+down_revision = '67ab88f410ce'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('username', table_name='bdx_feed')
+    op.alter_column('website_keywordcorpus', 'corpus',
+               existing_type=mysql.LONGBLOB(),
+               type_=LongText(),
+               existing_nullable=False)
+    op.alter_column('website_keywordcorpus', 'rawtext',
+               existing_type=mysql.LONGBLOB(),
+               type_=LongText(),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('website_keywordcorpus', 'rawtext',
+               existing_type=LongText(),
+               type_=mysql.LONGBLOB(),
+               existing_nullable=False)
+    op.alter_column('website_keywordcorpus', 'corpus',
+               existing_type=LongText(),
+               type_=mysql.LONGBLOB(),
+               existing_nullable=False)
+    op.create_index('username', 'bdx_feed', ['username'], unique=True)
+    # ### end Alembic commands ###
diff --git a/alembic/versions/e84b73490a04_altered_go_cloud_table.py b/alembic/versions/e84b73490a04_altered_go_cloud_table.py
@@ -0,0 +1,74 @@
+"""altered go_cloud table
+
+Revision ID: e84b73490a04
+Revises: c934ef207888
+Create Date: 2024-04-28 20:14:51.670740
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+import sqlalchemy_utils  # type: ignore
+from app.db.custom_types import LongText
+
+# revision identifiers, used by Alembic.
+revision = 'e84b73490a04'
+down_revision = 'c934ef207888'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('go_cloud', 'project_id',
+               existing_type=mysql.VARCHAR(length=704),
+               type_=sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(length=472),
+               existing_nullable=False)
+    op.alter_column('go_cloud', 'project_number',
+               existing_type=mysql.VARCHAR(length=704),
+               type_=sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(length=472),
+               existing_nullable=False)
+    op.alter_column('go_cloud', 'service_account',
+               existing_type=mysql.VARCHAR(length=704),
+               nullable=True)
+    op.create_unique_constraint(None, 'go_cloud', ['project_number'])
+    op.create_unique_constraint(None, 'go_cloud', ['project_id'])
+    op.create_unique_constraint(None, 'go_cloud', ['service_account'])
+    op.drop_column('go_cloud', 'api_key')
+    op.alter_column('website_keywordcorpus', 'corpus',
+               existing_type=mysql.LONGBLOB(),
+               type_=LongText(),
+               existing_nullable=False)
+    op.alter_column('website_keywordcorpus', 'rawtext',
+               existing_type=mysql.LONGBLOB(),
+               type_=LongText(),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('website_keywordcorpus', 'rawtext',
+               existing_type=LongText(),
+               type_=mysql.LONGBLOB(),
+               existing_nullable=False)
+    op.alter_column('website_keywordcorpus', 'corpus',
+               existing_type=LongText(),
+               type_=mysql.LONGBLOB(),
+               existing_nullable=False)
+    op.add_column('go_cloud', sa.Column('api_key', mysql.VARCHAR(length=704), nullable=False))
+    op.drop_constraint(None, 'go_cloud', type_='unique')  # type: ignore
+    op.drop_constraint(None, 'go_cloud', type_='unique')  # type: ignore
+    op.drop_constraint(None, 'go_cloud', type_='unique')  # type: ignore
+    op.alter_column('go_cloud', 'service_account',
+               existing_type=mysql.VARCHAR(length=704),
+               nullable=False)
+    op.alter_column('go_cloud', 'project_number',
+               existing_type=sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(length=472),
+               type_=mysql.VARCHAR(length=704),
+               existing_nullable=False)
+    op.alter_column('go_cloud', 'project_id',
+               existing_type=sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(length=472),
+               type_=mysql.VARCHAR(length=704),
+               existing_nullable=False)
+    # ### end Alembic commands ###
diff --git a/app/api/deps/__init__.py b/app/api/deps/__init__.py
@@ -13,8 +13,11 @@
     FetchWebPageSpeedInsightOr404,
     FetchWebsiteKeywordCorpusOr404,
     FetchWebsiteOr404,
+    get_bdx_feed_404,
     get_client_or_404,
     get_ga4_property_404,
+    get_ga4_stream_404,
+    get_go_cloud_404,
     get_note_or_404,
     get_sharpspring_404,
     get_user_or_404,
@@ -37,6 +40,7 @@
     CommonClientWebsiteQueryParams,
     CommonUserClientQueryParams,
     CommonUserQueryParams,
+    CommonWebsiteGa4QueryParams,
     CommonWebsiteKeywordCorpusQueryParams,
     CommonWebsiteMapQueryParams,
     CommonWebsitePageQueryParams,
@@ -48,6 +52,7 @@
     GetPublicQueryParams,
     GetUserClientQueryParams,
     GetUserQueryParams,
+    GetWebsiteGa4QueryParams,
     GetWebsiteKeywordCorpusQueryParams,
     GetWebsiteMapQueryParams,
     GetWebsitePageQueryParams,
@@ -78,8 +83,11 @@
     "FetchWebsiteOr404",
     "get_client_or_404",
     "get_note_or_404",
+    "get_bdx_feed_404",
     "get_sharpspring_404",
+    "get_go_cloud_404",
     "get_ga4_property_404",
+    "get_ga4_stream_404",
     "get_user_or_404",
     "get_website_map_or_404",
     "get_website_or_404",
@@ -108,8 +116,10 @@
     "CommonWebsiteMapQueryParams",
     "CommonWebsitePageSpeedInsightsQueryParams",
     "CommonWebsiteKeywordCorpusQueryParams",
+    "CommonWebsiteGa4QueryParams",
     "GetPublicQueryParams",
     "GetUserQueryParams",
+    "GetWebsiteGa4QueryParams",
     "GetClientQueryParams",
     "GetUserClientQueryParams",
     "GetWebsiteQueryParams",

diff --git a/app/api/deps/get_db_items.py b/app/api/deps/get_db_items.py
@@ -5,8 +5,11 @@
 
 from app.api.deps.get_db import AsyncDatabaseSession
 from app.api.exceptions import (
+    BdxFeedNotExists,
     ClientNotExists,
     Ga4PropertyNotExists,
+    Ga4StreamNotExists,
+    GoCloudPropertyNotExists,
     NoteNotExists,
     SharpspringNotExists,
     UserNotExists,
@@ -18,8 +21,11 @@
 )
 from app.core.utilities.uuids import parse_id
 from app.crud import (
+    BdxFeedRepository,
     ClientRepository,
     GoAnalytics4PropertyRepository,
+    GoAnalytics4StreamRepository,
+    GoCloudPropertyRepository,
     NoteRepository,
     SharpspringRepository,
     UserRepository,
@@ -30,8 +36,11 @@
     WebsiteRepository,
 )
 from app.models import (
+    BdxFeed,
     Client,
     GoAnalytics4Property,
+    GoAnalytics4Stream,
+    GoCloudProperty,
     Note,
     Sharpspring,
     User,
@@ -183,6 +192,19 @@ async def get_website_page_kwc_or_404(
 ]
 
 
+async def get_bdx_feed_404(
+    db: AsyncDatabaseSession,
+    bdx_id: Any,
+) -> BdxFeed | None:
+    """Parses uuid/int and fetches bdx_feed by id."""
+    parsed_id: UUID = parse_id(bdx_id)
+    bdx_repo: BdxFeedRepository = BdxFeedRepository(session=db)
+    bdx_feed: BdxFeed | None = await bdx_repo.read(parsed_id)
+    if bdx_feed is None:
+        raise BdxFeedNotExists()
+    return bdx_feed
+
+
 async def get_sharpspring_404(
     db: AsyncDatabaseSession,
     ss_id: Any,
@@ -196,6 +218,19 @@ async def get_sharpspring_404(
     return ss_acct
 
 
+async def get_go_cloud_404(
+    db: AsyncDatabaseSession,
+    go_cloud_id: Any,
+) -> GoCloudProperty | None:
+    """Parses uuid/int and fetches google cloud property by id."""
+    parsed_id: UUID = parse_id(go_cloud_id)
+    go_cloud_repo: GoCloudPropertyRepository = GoCloudPropertyRepository(session=db)
+    go_cloud_acct: Sharpspring | None = await go_cloud_repo.read(parsed_id)
+    if go_cloud_acct is None:
+        raise GoCloudPropertyNotExists()
+    return go_cloud_acct
+
+
 async def get_ga4_property_404(
     db: AsyncDatabaseSession,
     ga4_id: Any,
@@ -209,3 +244,18 @@ async def get_ga4_property_404(
     if ga4_property is None:
         raise Ga4PropertyNotExists()
     return ga4_property
+
+
+async def get_ga4_stream_404(
+    db: AsyncDatabaseSession,
+    ga4_stream_id: Any,
+) -> GoAnalytics4Stream | None:
+    """Parses uuid/int and fetches ga4 stream by id."""
+    parsed_id: UUID = parse_id(ga4_stream_id)
+    ga4_stream_repo: GoAnalytics4StreamRepository = GoAnalytics4StreamRepository(
+        session=db
+    )
+    ga4_stream: GoAnalytics4Stream | None = await ga4_stream_repo.read(parsed_id)
+    if ga4_stream is None:
+        raise Ga4StreamNotExists()
+    return ga4_stream
diff --git a/app/api/deps/get_query.py b/app/api/deps/get_query.py
@@ -94,6 +94,19 @@ def __init__(self, strategy: List[str] | None = None):
         self.strategy: List[str] | None = q_devices
 
 
+class Ga4QueryParams:
+    def __init__(self, ga4_id: str | None = None):
+        q_ga4_id: UUID4 | None
+        try:
+            q_ga4_id = None if not ga4_id else parse_id(ga4_id)
+        except InvalidID:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail="Invalid ga4 property ID",
+            )
+        self.ga4_id: UUID4 | None = q_ga4_id
+
+
 # compound query classes
 
 
@@ -314,3 +327,27 @@ def __init__(
 GetWebsiteKeywordCorpusQueryParams = Annotated[
     CommonWebsiteKeywordCorpusQueryParams, Depends()
 ]
+
+
+class CommonWebsiteGa4QueryParams(
+    PageParamsFromQuery, WebsiteIdQueryParams, Ga4QueryParams
+):
+    def __init__(
+        self,
+        page: Annotated[int | None, Query(ge=1)] = 1,
+        size: Annotated[
+            int | None,
+            Query(
+                ge=1,
+                le=settings.api.query_limit_rows_max,
+            ),
+        ] = settings.api.query_limit_rows_default,
+        website_id: Annotated[str | None, Query()] = None,
+        ga4_id: Annotated[str | None, Query()] = None,
+    ):
+        PageParamsFromQuery.__init__(self, page, size)
+        WebsiteIdQueryParams.__init__(self, website_id)
+        Ga4QueryParams.__init__(self, ga4_id)
+
+
+GetWebsiteGa4QueryParams = Annotated[CommonWebsiteGa4QueryParams, Depends()]