From fa101d49fd7ee85b2bb07b34fdb61ae0cd9f914a Mon Sep 17 00:00:00 2001 From: Alexander Pyatkov Date: Thu, 2 Mar 2023 13:11:09 +0400 Subject: [PATCH] Added removal functionality. Added args and help. Added README.md --- README.md | 30 ++++++++++++++++++++++++++ main.py | 55 ++++++++++++++++++++++++++++++++---------------- requirements.txt | 4 ++++ 3 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 README.md create mode 100644 requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..5c1f616 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# Google API Indexer + +This script can automatically update and delete URLs via Google Indexing API. Store your service-accounts's json keys in folder `./json_keys` and run script. + +### Requirements + +- first you need to install requirements: +`pip install -r requirements.txt` + +### Usage + +``` +python3 main.py --help +usage: main.py [-h] [-d] [-i INPUT] [-t {txt_file,database}] [-H HOST] [-U USER] [-P PASSWORD] [-D DATABASE] + +options: + -h, --help show this help message and exit + -d, --delete Delete URLs + -i INPUT, --input INPUT + Path to .csv file with URLs (default ./urls.csv) + -t {txt_file,database}, --outtype {txt_file,database} + Type of result output (default txt_file). Output can be written to a file result.txt or to a MySQL + database. If 'database' is selected then host, user, password, and database-name must be specified + -H HOST, --host HOST Database's host to connect (default 127.0.0.1) + -U USER, --user USER Database's user to connect + -P PASSWORD, --password PASSWORD + Database user's password + -D DATABASE, --database DATABASE + Database to connect +``` \ No newline at end of file diff --git a/main.py b/main.py index 458aff4..b6908e0 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +import argparse import datetime from oauth2client.service_account import ServiceAccountCredentials import httplib2 @@ -5,27 +6,18 @@ import os from script_mysql import MySQLi -from config import * -""" -pip install google-api-python-client oauth2client -pip install --upgrade oauth2client -""" - def write_result(work_type, url, date): if work_type == 'database': - db = MySQLi(host, user, password, database_home) + db = MySQLi(args.host, args.user, args.password, args.database) db.commit("INSERT INTO indexing_api (url, date) VALUES (%s, %s)", url_new, datetime.date.today()) elif work_type == 'txt_file': with open('result.txt', 'a', encoding='utf-8') as result_file: string_write = f"{url};{date}\n" result_file.write(string_write) -SCOPES = ["https://www.googleapis.com/auth/indexing"] - - -def indexURL2(u, http): +def indexURL2(u, http, action): ENDPOINT = "https://indexing.googleapis.com/v3/urlNotifications:publish" - content = {'url': u.strip(), 'type': "URL_UPDATED"} + content = {'url': u.strip(), 'type': action} json_ctn = json.dumps(content) response, content = http.request(ENDPOINT, method="POST", body=json_ctn) result = json.loads(content.decode()) @@ -35,7 +27,7 @@ def indexURL2(u, http): result["error"]["message"])) return "Error({} - {}): {}".format(result["error"]["code"], result["error"]["status"], result["error"]["message"]) - else: + elif action == "URL_UPDATED": print("urlNotificationMetadata.url: {}".format(result["urlNotificationMetadata"]["url"])) print("urlNotificationMetadata.latestUpdate.url: {}".format( result["urlNotificationMetadata"]["latestUpdate"]["url"])) @@ -44,18 +36,44 @@ def indexURL2(u, http): print("urlNotificationMetadata.latestUpdate.notifyTime: {}".format( result["urlNotificationMetadata"]["latestUpdate"]["notifyTime"])) return "OK" + elif action == "URL_DELETED": + print("urlNotificationMetadata.url: {}".format(result["urlNotificationMetadata"]["url"])) + print("urlNotificationMetadata.latestRemove.url: {}".format( + result["urlNotificationMetadata"]["latestRemove"]["url"])) + print("urlNotificationMetadata.latestRemove.type: {}".format( + result["urlNotificationMetadata"]["latestRemove"]["type"])) + print("urlNotificationMetadata.latestRemove.notifyTime: {}".format( + result["urlNotificationMetadata"]["latestRemove"]["notifyTime"])) + return "OK" +SCOPES = ["https://www.googleapis.com/auth/indexing"] count_urls = 0 + +args_pr = argparse.ArgumentParser() +args_pr.add_argument("-d", "--delete", action='store_const', const=True, required=False, help="Delete URLs") +args_pr.add_argument("-i", "--input", required=False, type=str, default="urls.csv", help="Path to .csv file with URLs (default ./urls.csv)") +args_pr.add_argument("-t", "--outtype", required=False, type=str, choices=["txt_file", "database"], default="txt_file", help="Type of result output (default txt_file). \ + Output can be written to a file result.txt or to a MySQL database. \ + If 'database' is selected then host, user, password, and database-name must be specified") +args_pr.add_argument("-H", "--host", required=False, type=str, default="127.0.0.1", help="Database's host to connect (default 127.0.0.1)") +args_pr.add_argument("-U", "--user", required=False, type=str, help="Database's user to connect") +args_pr.add_argument("-P", "--password", required=False, type=str, help="Database user's password") +args_pr.add_argument("-D", "--database", required=False, type=str, help="Database to connect") +args = args_pr.parse_args() + +action = "URL_UPDATED" +if args.delete: action = "URL_DELETED" + for root, dirs, files in os.walk("json_keys"): for json_key_path_name in files: json_key = 'json_keys/' + json_key_path_name credentials = ServiceAccountCredentials.from_json_keyfile_name(json_key, scopes=SCOPES) http = credentials.authorize(httplib2.Http()) - a_file = open("urls.csv", "r") # get list of lines + a_file = open(args.input, "r") # get list of lines urls = a_file.readlines() a_file.close() - new_file = open("urls.csv", "w") + new_file = open(args.input, "w") flag = False request_google_api = '' for url in urls: @@ -63,7 +81,7 @@ def indexURL2(u, http): if flag: new_file.write(url) else: - request_google_api = indexURL2(url_new, http) + request_google_api = indexURL2(url_new, http, action) if 'Error' in request_google_api: flag = True @@ -71,9 +89,10 @@ def indexURL2(u, http): request_google_api = '' else: if not flag: - write_result('txt_file', url_new, datetime.date.today()) + write_result(args.outtype, url_new, datetime.date.today()) count_urls += 1 new_file.close() -print("Отправлено на индексацию: " + str(count_urls) + " шт.") +if action == "URL_UPDATED": print("Отправлено на индексацию: " + str(count_urls) + " шт.") +else: print("Отправлено на деиндексацию: " + str(count_urls) + " шт.") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9f3fad7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +google-api-python-client +oauth2client +argparse +mysql-connector-python==8.0.32 \ No newline at end of file