Multi wallet & collectibles (#1301)

* Multi Wallet Support (#1296)

* Validate CN for associated_wallest in user metadata

* Add associated wallets to Discovery DB

* Add assocaited wallet to discovery schema

* Update discovery indexing for user associated wallets

* Add discovery endpoints for wallet - id

* Add wallets schema to libs user validation

* Update schema and user balance for associated wallet  (#1299)

* Update associated_wallet schema and update disc. user balance on wallet update

* Update schema in libs for associated wallets

* Update validate metadata in creator for assciated wallet

* Fix CN test

* Add another check to CN metadata upload

* Lint discovery

* Update user schema validator in discovery and libs

* Feature collectibles metadata multihash (#1292)

* Support collectibles in libs

* Support metadata multihash for collectibles in discovery

* Add has_collectibles indexed field to user object

* Fix wallet object check

* Clean up

* Do not check metadata from ipfs every time

* Fix alembic collisions

* Handle the null cases in user metadata indexing

* Forgo image size changes

* Fix clean metadata for new fields

Co-authored-by: Joseph Lee <joeylee0925@gmail.com>
Co-authored-by: Raymond Jacobson <ray@audius.co>
Co-authored-by: Saliou Diallo <saliou@audius.co>

creator-node/src/routes/audiusUsers.js

@@ -6,6 +6,7 @@ const models = require('../models')
 const { saveFileFromBufferToIPFSAndDisk } = require('../fileManager')
 const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers')
 const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils')
+const validateMetadata = require('../utils/validateAudiusUserMetadata')
 const {
   authMiddleware,
   syncLockMiddleware,
@@ -26,6 +27,10 @@ module.exports = function (app) {
     const metadataJSON = req.body.metadata
     const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON))
     const cnodeUserUUID = req.session.cnodeUserUUID
+    let isValidMetadata = validateMetadata(req, metadataJSON)
+    if (!isValidMetadata) {
+      return errorResponseBadRequest('Invalid User Metadata')
+    }
 
     // Save file from buffer to IPFS and disk
     let multihash, dstPath

creator-node/src/utils/validateAudiusUserMetadata.js

@@ -0,0 +1,42 @@
+const Web3 = require('web3')
+const web3 = new Web3()
+
+/**
+ * Validates that the user id was signed by the associated wallets
+ */
+const validateAssociatedWallets = (metadataJSON) => {
+  if ('user_id' in metadataJSON && 'associated_wallets' in metadataJSON) {
+    const userId = metadataJSON['user_id']
+    const walletMappings = metadataJSON['associated_wallets']
+    if (!walletMappings) return true
+    if (typeof walletMappings !== 'object') return true
+    const message = `AudiusUserID:${userId}`
+    return Object.keys(walletMappings).every(wallet => {
+      if (
+        typeof walletMappings[wallet] !== 'object' ||
+        walletMappings[wallet] === null ||
+        !('signature' in walletMappings[wallet])
+      ) return false
+      const signature = walletMappings[wallet].signature
+      const recoveredWallet = web3.eth.accounts.recover(message, signature)
+      return recoveredWallet === wallet
+    })
+  }
+  return true
+}
+
+const validateMetadata = (req, metadataJSON) => {
+  // Check associated wallets
+  if (typeof metadataJSON !== 'object' || metadataJSON === null) {
+    return false
+  } else if (!validateAssociatedWallets(metadataJSON)) {
+    req.logger.info('Associated Wallets do not match signatures')
+    return false
+  }
+
+  // TODO: Add more validation checks
+  return true
+}
+
+module.exports = validateMetadata
+module.exports.validateAssociatedWallets = validateAssociatedWallets

creator-node/test/audiusUsers.test.js

@@ -131,17 +131,17 @@ describe('Test AudiusUsers with real IPFS', function () {
     assert.deepStrictEqual(resp.body.error, 'Internal server error')
   })
 
-  it('should throw error response if saving metadata fails', async function () {
+  it('should throw 400 bad request response if metadata validation fails', async function () {
     sinon.stub(ipfs, 'add').rejects(new Error('ipfs add failed!'))
 
     const metadata = { metadata: 'spaghetti' }
     const resp = await request(app)
       .post('/audius_users/metadata')
       .set('X-Session-ID', session.sessionToken)
       .send(metadata)
-      .expect(500)
+      .expect(400)
 
-    assert.deepStrictEqual(resp.body.error, 'saveFileFromBufferToIPFSAndDisk op failed: Error: ipfs add failed!')
+    assert.deepStrictEqual(resp.body.error, 'Invalid User Metadata')
   })
 
   it('successfully creates Audius user (POST /audius_users/metadata)', async function () {

discovery-provider/alembic/versions/5a2f90f7def1_add_has_collectibles.py

@@ -0,0 +1,24 @@
+"""add_has_collectibles
+
+Revision ID: 5a2f90f7def1
+Revises: f0a108d978ed
+Create Date: 2021-03-11 17:58:56.428933
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '5a2f90f7def1'
+down_revision = 'f0a108d978ed'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('users', sa.Column('has_collectibles', sa.Boolean(), server_default='false', nullable=False))
+
+
+def downgrade():
+    op.drop_column('users', 'has_collectibles')

discovery-provider/alembic/versions/f0a108d978ed_add_associated_wallets.py

@@ -0,0 +1,34 @@
+"""add associated wallets
+
+Revision ID: f0a108d978ed
+Revises: c967ae0fcaf6
+Create Date: 2021-03-09 15:34:32.229219
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f0a108d978ed'
+down_revision = 'c967ae0fcaf6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('associated_wallets',
+        sa.Column('id', sa.Integer(), nullable=False, autoincrement=True),
+        sa.Column('user_id', sa.Integer(), nullable=False),
+        sa.Column('wallet', sa.String(), nullable=False),
+        sa.Column('blockhash', sa.String(), nullable=False),
+        sa.Column('blocknumber', sa.Integer(), nullable=False),
+        sa.Column('is_current', sa.Boolean(), nullable=False),
+        sa.Column('is_delete', sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_associated_wallets_user_id'), 'associated_wallets', ['user_id', 'is_current'], unique=False)
+    op.create_index(op.f('ix_associated_wallets_wallet'), 'associated_wallets', ['wallet', 'is_current'], unique=False)
+
+def downgrade():
+    op.drop_table('associated_wallets')

discovery-provider/src/api/v1/models/users.py

@@ -1,6 +1,14 @@
 from flask_restx import fields
 from .common import ns
 
+associated_wallets = ns.model("associated_wallets", {
+    "wallets": fields.List(fields.String, required=True)
+})
+
+encoded_user_id = ns.model("encoded_user_id", {
+    "user_id": fields.String(allow_null=True)
+})
+
 profile_picture = ns.model("profile_picture", {
     "150x150": fields.String,
     "480x480": fields.String,
@@ -26,7 +34,7 @@
     "playlist_count": fields.Integer(required=True),
     "profile_picture": fields.Nested(profile_picture, allow_null=True),
     "repost_count": fields.Integer(required=True),
-    "track_count": fields.Integer(required=True),
+    "track_count": fields.Integer(required=True)
 })
 
 user_model_full = ns.clone("user_full", user_model, {
@@ -43,5 +51,7 @@
     "cover_photo_sizes": fields.String,
     "cover_photo_legacy": fields.String,
     "profile_picture_sizes": fields.String,
-    "profile_picture_legacy": fields.String
+    "profile_picture_legacy": fields.String,
+    "metadata_multihash": fields.String,
+    "has_collectibles": fields.Boolean(required=True)
 })

discovery-provider/src/api/v1/users.py

@@ -3,7 +3,7 @@
 from src.queries.get_repost_feed_for_user import get_repost_feed_for_user
 from flask_restx import Resource, Namespace, fields, reqparse
 from src.api.v1.models.common import favorite
-from src.api.v1.models.users import user_model, user_model_full
+from src.api.v1.models.users import user_model, user_model_full, associated_wallets, encoded_user_id
 
 from src.queries.get_saves import get_saves
 from src.queries.get_users import get_users
@@ -13,10 +13,12 @@
 from src.queries.get_followees_for_user import get_followees_for_user
 from src.queries.get_followers_for_user import get_followers_for_user
 from src.queries.get_top_user_track_tags import get_top_user_track_tags
+from src.queries.get_associated_user_wallet import get_associated_user_wallet
+from src.queries.get_associated_user_id import get_associated_user_id
 
 from src.api.v1.helpers import abort_not_found, decode_with_abort, extend_activity, extend_favorite, extend_track, \
     extend_user, format_limit, format_offset, get_current_user_id, make_full_response, make_response, search_parser, success_response, abort_bad_request_param, \
-    get_default_max
+    get_default_max, encode_int_id
 from .models.tracks import track, track_full
 from .models.activities import activity_model, activity_model_full
 from src.utils.redis_cache import cache
@@ -625,3 +627,48 @@ def get(self):
         top_users = get_top_genre_users(get_top_genre_users_args)
         users = list(map(extend_user, top_users['users']))
         return success_response(users)
+
+associated_wallet_route_parser = reqparse.RequestParser()
+associated_wallet_route_parser.add_argument('id', required=True)
+associated_wallet_response = make_response("associated_wallets_response", ns, fields.Nested(associated_wallets))
+@ns.route("/associated_wallets")
+class UserIdByAssociatedWallet(Resource):
+    @ns.expect(associated_wallet_route_parser)
+    @ns.doc(
+        id="""Get the User's id by associated wallet""",
+        params={'id': 'Encoded User ID'},
+        responses={
+            200: 'Success',
+            400: 'Bad request',
+            500: 'Server error'
+        }
+    )
+    @ns.marshal_with(associated_wallet_response)
+    @cache(ttl_sec=10)
+    def get(self):
+        args = associated_wallet_route_parser.parse_args()
+        user_id = decode_with_abort(args.get('id'), ns)
+        wallets = get_associated_user_wallet({ "user_id": user_id})
+        return success_response({ "wallets": wallets })
+
+user_associated_wallet_route_parser = reqparse.RequestParser()
+user_associated_wallet_route_parser.add_argument('associated_wallet', required=True)
+user_associated_wallet_response = make_response("user_associated_wallet_response", ns, fields.Nested(encoded_user_id))
+@ns.route("/id")
+class AssociatedWalletByUserId(Resource):
+    @ns.expect(user_associated_wallet_route_parser)
+    @ns.doc(
+        id="""Get the User's associated wallets""",
+        params={'associated_wallet': 'Wallet address'},
+        responses={
+            200: 'Success',
+            400: 'Bad request',
+            500: 'Server error'
+        }
+    )
+    @ns.marshal_with(user_associated_wallet_response)
+    @cache(ttl_sec=1)
+    def get(self):
+        args = user_associated_wallet_route_parser.parse_args()
+        user_id = get_associated_user_id({ "wallet" :args.get('associated_wallet') })
+        return success_response({ "user_id": encode_int_id(user_id) if user_id else None })

discovery-provider/src/models.py

@@ -156,6 +156,7 @@ class User(Base):
     primary_id = Column(Integer, nullable=True)
     secondary_ids = Column(postgresql.ARRAY(Integer), nullable=True)
     replica_set_update_signer = Column(String, nullable=True)
+    has_collectibles = Column(Boolean, nullable=False, default=False, server_default='false')
 
     # Primary key has to be combo of all 3 is_current/creator_id/blockhash
     PrimaryKeyConstraint(is_current, user_id, blockhash)
@@ -754,3 +755,22 @@ def __repr__(self):
         return f"<UserBalance(\
 user_id={self.user_id},\
 balance={self.balance}>"
+
+class AssociatedWallet(Base):
+    __tablename__ = "associated_wallets"
+    blockhash = Column(String, ForeignKey("blocks.blockhash"), nullable=False)
+    blocknumber = Column(Integer, ForeignKey("blocks.number"), nullable=False)
+    is_current = Column(Boolean, nullable=False)
+    is_delete = Column(Boolean, nullable=False)
+    id = Column(Integer, nullable=False, primary_key=True)
+    user_id = Column(Integer, nullable=False, index=True)
+    wallet = Column(String, nullable=False, index=True)
+
+    def __repr__(self):
+        return f"<AssociatedWallet(blockhash={self.blockhash},\
+blocknumber={self.blocknumber},\
+is_current={self.is_current},\
+is_delete={self.is_delete},\
+id={self.id},\
+user_id={self.user_id},\
+wallet={self.wallet})>"

discovery-provider/src/queries/get_associated_user_id.py

@@ -0,0 +1,28 @@
+import logging
+
+from src.models import AssociatedWallet
+from src.utils import db_session
+
+logger = logging.getLogger(__name__)
+
+def get_associated_user_id(args):
+    """
+    Returns a user_id the associated wallet
+
+    Args:
+        args: dict The parsed args from the request
+        args.wallet: string The wallet to find associated with an user id
+
+    Returns:
+        number representing the user id
+    """
+    db = db_session.get_db_read_replica()
+    with db.scoped_session() as session:
+        user_id = (
+            session.query(AssociatedWallet.user_id)
+            .filter(AssociatedWallet.is_current == True)
+            .filter(AssociatedWallet.is_delete == False)
+            .filter(AssociatedWallet.wallet == args.get('wallet'))
+            .first()
+        )
+        return user_id[0] if user_id else None

discovery-provider/src/queries/get_associated_user_wallet.py

@@ -0,0 +1,29 @@
+import logging
+
+from src.models import AssociatedWallet
+from src.utils import db_session
+
+logger = logging.getLogger(__name__)
+
+
+def get_associated_user_wallet(args):
+    """
+    Returns a list of associated wallets
+
+    Args:
+        args: dict The parsed args from the request
+        args.user_id: number The blockchain user id
+
+    Returns:
+        Array of strings representing the user's associated wallets
+    """
+    db = db_session.get_db_read_replica()
+    with db.scoped_session() as session:
+        user_wallet = (
+            session.query(AssociatedWallet.wallet)
+            .filter(AssociatedWallet.is_current == True)
+            .filter(AssociatedWallet.is_delete == False)
+            .filter(AssociatedWallet.user_id == args.get('user_id'))
+            .all()
+        )
+        return [wallet for [wallet] in user_wallet]