New command: twitter-to-sqlite lists, refs #43

README.md

@@ -126,6 +126,16 @@ You can also use the `--screen_name` or `--user_id` arguments to retrieve favori
 
     $ twitter-to-sqlite favorites faves-obama.db --screen_name=BarackObama
 
+## Retrieving Twitter lists
+
+The `lists` command retrieves all of the lists belonging to one or more users.
+
+    $ twitter-to-sqlite lists lists.db simonw dogsheep
+
+This command also accepts the `--sql` and `--attach` and `--ids` options.
+
+To additionally fetch the list of members for each list, use `--members`.
+
 ## Retrieving Twitter list memberships
 
 The `list-members` command can be used to retrieve details of one or more Twitter lists, including all of their members.

setup.py

@@ -27,7 +27,7 @@ def get_long_description():
         twitter-to-sqlite=twitter_to_sqlite.cli:cli
     """,
     install_requires=[
-        "sqlite-utils~=2.4.2",
+        "sqlite-utils>=2.4.2",
         "requests-oauthlib~=1.2.0",
         "python-dateutil",
     ],

twitter_to_sqlite/cli.py

@@ -509,6 +509,49 @@ def statuses_lookup(db_path, identifiers, attach, sql, auth, skip_existing, sile
                 bar.update(len(batch))
 
 
+@cli.command(name="lists")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@add_identifier_options
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+@click.option("--ids", is_flag=True, help="Treat input as user IDs, not screen_names")
+@click.option("--members", is_flag=True, help="Retrieve members for each list")
+def lists(db_path, identifiers, attach, sql, auth, ids, members):
+    "Fetch lists belonging to specified users"
+    auth = json.load(open(auth))
+    session = utils.session_for_auth(auth)
+    db = utils.open_database(db_path)
+    identifiers = utils.resolve_identifiers(db, identifiers, attach, sql)
+    # Make sure we have saved these users to the database
+    for batch in utils.fetch_user_batches(session, identifiers, ids):
+        utils.save_users(db, batch)
+    first = True
+    for identifier in identifiers:
+        if ids:
+            kwargs = {"user_id": identifier}
+        else:
+            kwargs = {"screen_name": identifier}
+        fetched_lists = utils.fetch_lists(db, session, **kwargs)
+        if members:
+            for new_list in fetched_lists:
+                utils.fetch_and_save_list(
+                    db, session, new_list["full_name"].rstrip("@")
+                )
+        if not first:
+            # Rate limit is one per minute
+            first = False
+            time.sleep(60)
+
+
 @cli.command(name="list-members")
 @click.argument(
     "db_path",

twitter_to_sqlite/utils.py

@@ -93,6 +93,23 @@ def fetch_user_list(session, cursor, user_id=None, screen_name=None, noun="follo
     return r.headers, r.json()
 
 
+def fetch_lists(db, session, user_id=None, screen_name=None):
+    lists_url = "https://api.twitter.com/1.1/lists/ownerships.json"
+    args = user_args(user_id, screen_name)
+    args["count"] = 1000
+    fetched_lists = []
+    # For the moment we don't paginate
+    for list_row in session.get(lists_url, params=args).json()["lists"]:
+        del list_row["id_str"]
+        user = list_row.pop("user")
+        save_users(db, [user])
+        list_row["user"] = user["id"]
+        list_row["created_at"] = parser.parse(list_row["created_at"])
+        fetched_lists.append(list_row)
+    db["lists"].insert_all(fetched_lists, pk="id", foreign_keys=("user",), replace=True)
+    return fetched_lists
+
+
 def get_profile(db, session, user_id=None, screen_name=None):
     if not (user_id or screen_name):
         profile = session.get(