Merge pull request #210 from AdityaJ7/master

Added Flake8 to CI

Author: chavarera

.github/workflows/python-app.yml

@@ -0,0 +1,34 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python application
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    

projects/web page summation/utils/model.py

@@ -1,6 +1,6 @@
 import tensorflow as tf
 from tensorflow.contrib import rnn
-#from utils import get_init_embedding
+from utils import get_init_embedding
 
 
 class Model(object):

projects/web page summation/utils/prepare.py

@@ -1,19 +1,6 @@
-
-import wget
-import os
-import tarfile
 import gzip
-import zipfile
-import argparse
-
 
-#parser = argparse.ArgumentParser()
-#parser.add_argument("--glove", action="store_true")
-#args = parser.parse_args()
-
-# Extract data file
-#with tarfile.open(default_path + "sumdata/train/summary.tar.gz", "r:gz") as tar:
-#    tar.extractall()
+default_path = '.'
 
 with gzip.open(default_path + "sumdata/train/train.article.txt.gz", "rb") as gz:
     with open(default_path + "sumdata/train/train.article.txt", "wb") as out:
@@ -22,18 +9,3 @@
 with gzip.open(default_path + "sumdata/train/train.title.txt.gz", "rb") as gz:
     with open(default_path + "sumdata/train/train.title.txt", "wb") as out:
         out.write(gz.read())
-
-        
-#if args.glove:
-#    glove_dir = "glove"
-#    glove_url = "https://nlp.stanford.edu/data/wordvecs/glove.42B.300d.zip"
-#
-#    if not os.path.exists(glove_dir):
-#        os.mkdir(glove_dir)
-#
-#    # Download glove vector
-#    wget.download(glove_url, out=glove_dir)
-#
-#    # Extract glove file
-#    with zipfile.ZipFile(os.path.join("glove", "glove.42B.300d.zip"), "r") as z:
-#        z.extractall(glove_dir)

projects/web page summation/utils/summarize.py

@@ -1,10 +1,6 @@
-# -*- coding: utf-8 -*-
-# load Dependancies
-
 from __future__ import absolute_import
 from __future__ import division, print_function, unicode_literals
 from sumy.parsers.html import HtmlParser
-from sumy.parsers.plaintext import PlaintextParser
 from sumy.nlp.tokenizers import Tokenizer
 from sumy.summarizers.lex_rank import LexRankSummarizer as Summarizer
 from sumy.nlp.stemmers import Stemmer
@@ -13,25 +9,22 @@
 
 
 def summarize(url=None, LANGUAGE='English', SENTENCES_COUNT=2):
-        parser = HtmlParser.from_url(url, Tokenizer(LANGUAGE))
-        # or for plain text files
-        # parser = PlaintextParser.from_file("document.txt", Tokenizer(LANGUAGE))
-        stemmer = Stemmer(LANGUAGE)
-
-        summarizer = Summarizer(stemmer)
-        summarizer.stop_words = get_stop_words(LANGUAGE)
-        result = ''
-        for sentence in summarizer(parser.document, SENTENCES_COUNT):
+    parser = HtmlParser.from_url(url, Tokenizer(LANGUAGE))
+    stemmer = Stemmer(LANGUAGE)
+    summarizer = Summarizer(stemmer)
+    summarizer.stop_words = get_stop_words(LANGUAGE)
+    result = ''
+    for sentence in summarizer(parser.document, SENTENCES_COUNT):
+        result = result + ' ' + str(sentence)
+        try:
             result = result + ' ' + str(sentence)
-            try:
-                result = result + ' ' + str(sentence)
 
-            except:
-                print(
+        except:
+            print(
                     '\n\n Invalid Entry!, please Ensure you enter a valid web link \n\n')
-                sys.stdout.flush()
-                return (
+            sys.stdout.flush()
+            return (
                     '\n\n Invalid Entry!, please Ensure you enter a valid web link \n\n')
-        print('\n\n'+str(url)+'\n\n'+str(result))
-        sys.stdout.flush()
-        return result
+    print('\n\n'+str(url)+'\n\n'+str(result))
+    sys.stdout.flush()
+    return result

projects/web page summation/utils/test.py

@@ -1,13 +1,10 @@
 import tensorflow as tf
-import pickle
-#from model import Model
-#from utils import build_dict, build_dataset, batch_iter
-
-
-# with open("args.pickle", "rb") as f:
-#    args = pickle.load(f)
-
+from model import Model
+from utils import build_dict, build_dataset, batch_iter, get_text_list
+valid_article_path = '.'
+valid_title_path = '.'
 tf.reset_default_graph()
+default_path = '.'
 
 
 class args:
@@ -74,6 +71,6 @@ class args:
                     if word not in summary:
                         summary.append(word)
                 summary_array.append(" ".join(summary))
-                #print(" ".join(summary), file=f)
+                # print(" ".join(summary), file=f)
 
     print('Summaries have been generated')

projects/web page summation/utils/train.py

@@ -1,31 +1,10 @@
 import os
-import pickle
-import argparse
 import tensorflow as tf
 import time
+from model import Model
+from utils import build_dict, build_dataset, batch_iter
 start = time.perf_counter()
-#from model import Model
-#from utils import build_dict, build_dataset, batch_iter
-
-# Uncomment next 2 lines to suppress error and Tensorflow info verbosity. Or change logging levels
-# tf.logging.set_verbosity(tf.logging.FATAL)
-# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
-
-# def add_arguments(parser):
-#    parser.add_argument("--num_hidden", type=int, default=150, help="Network size.")
-#    parser.add_argument("--num_layers", type=int, default=2, help="Network depth.")
-#    parser.add_argument("--beam_width", type=int, default=10, help="Beam width for beam search decoder.")
-#    parser.add_argument("--glove", action="store_true", help="Use glove as initial word embedding.")
-#    parser.add_argument("--embedding_size", type=int, default=300, help="Word embedding size.")
-#
-#    parser.add_argument("--learning_rate", type=float, default=1e-3, help="Learning rate.")
-#    parser.add_argument("--batch_size", type=int, default=64, help="Batch size.")
-#    parser.add_argument("--num_epochs", type=int, default=10, help="Number of epochs.")
-#    parser.add_argument("--keep_prob", type=float, default=0.8, help="Dropout keep prob.")
-#
-#    parser.add_argument("--toy", action="store_true", help="Use only 50K samples of data")
-#
-#    parser.add_argument("--with_model", action="store_true", help="Continue from previously saved model")
+default_path = '.'
 
 
 class args:
@@ -48,20 +27,16 @@ class args:
 args.with_model = "store_true"
 
 
-#parser = argparse.ArgumentParser()
-# add_arguments(parser)
-#args = parser.parse_args()
-# with open("args.pickle", "wb") as f:
-#    pickle.dump(args, f)
-
 if not os.path.exists(default_path + "saved_model"):
     os.mkdir(default_path + "saved_model")
 else:
     # if args.with_model:
     old_model_checkpoint_path = open(
         default_path + 'saved_model/checkpoint', 'r')
     old_model_checkpoint_path = "".join(
-        [default_path + "saved_model/", old_model_checkpoint_path.read().splitlines()[0].split('"')[1]])
+        [
+            default_path + "saved_model/",
+            old_model_checkpoint_path.read().splitlines()[0].split('"')[1]])
 
 
 print("Building dictionary...")
@@ -98,9 +73,13 @@ class args:
             map(lambda x: list(x) + [word_dict["</s>"]], batch_y))
 
         batch_decoder_input = list(
-            map(lambda d: d + (summary_max_len - len(d)) * [word_dict["<padding>"]], batch_decoder_input))
+            map(
+                lambda d: d + (summary_max_len - len(d)) * [word_dict["<padding>"]],
+                batch_decoder_input))
         batch_decoder_output = list(
-            map(lambda d: d + (summary_max_len - len(d)) * [word_dict["<padding>"]], batch_decoder_output))
+            map(
+                lambda d: d + (summary_max_len - len(d)) * [word_dict["<padding>"]],
+                batch_decoder_output))
 
         train_feed_dict = {
             model.batch_size: len(batch_x),
@@ -112,7 +91,8 @@ class args:
         }
 
         _, step, loss = sess.run(
-            [model.update, model.global_step, model.loss], feed_dict=train_feed_dict)
+            [model.update,
+             model.global_step, model.loss], feed_dict=train_feed_dict)
 
         if step % 1000 == 0:
             print("step {0}: loss = {1}".format(step, loss))

projects/web page summation/utils/utils.py

@@ -2,11 +2,12 @@
 import collections
 import pickle
 import numpy as np
-from newspaper import Article
 from nltk.tokenize import word_tokenize
-from gensim.models.keyedvectors import KeyedVectors
-from gensim.test.utils import get_tmpfile
-from gensim.scripts.glove2word2vec import glove2word2vec
+default_path = '.'
+train_article_path = '.'
+train_title_path = '.'
+valid_article_path = '.'
+
 
 def clean_str(sentence):
     sentence = re.sub("[#.]+", "#", sentence)
@@ -55,7 +56,8 @@ def build_dict(step, toy=False):
     return word_dict, reversed_dict, article_max_len, summary_max_len
 
 
-def build_dataset(step, word_dict, article_max_len, summary_max_len, toy=False):
+def build_dataset(
+        step, word_dict, article_max_len, summary_max_len, toy=False):
     if step == "train":
         article_list = get_text_list(train_article_path, toy)
         title_list = get_text_list(train_title_path, toy)
@@ -91,11 +93,11 @@ def batch_iter(inputs, outputs, batch_size, num_epochs):
 
 
 def get_init_embedding(reversed_dict, embedding_size):
-    #glove_file = default_path + "glove/glove.6B.300d.txt"
-    #word2vec_file = get_tmpfile(default_path + "word2vec_format.vec")
-    #glove2word2vec(glove_file, word2vec_file)
+    # glove_file = default_path + "glove/glove.6B.300d.txt"
+    # word2vec_file = get_tmpfile(default_path + "word2vec_format.vec")
+    # glove2word2vec(glove_file, word2vec_file)
     print("Loading Glove vectors...")
-    #word_vectors = KeyedVectors.load_word2vec_format(word2vec_file)
+    # word_vectors = KeyedVectors.load_word2vec_format(word2vec_file)
 
     with open(default_path + "glove/model_glove_300.pkl", 'rb') as handle:
         word_vectors = pickle.load(handle)

projects/write a script to download a random image from unsplash and set it as wallpaper/background_windows.py

@@ -5,7 +5,7 @@
 import sys
 
 url = "https://source.unsplash.com/random"
-filename = "random.jpg"
+file_name = "random.jpg"
 
 def is_64bit():
     return sys.maxsize > 2 ** 32
@@ -32,7 +32,7 @@ def setup(pathtofile,version):
 
 if __name__ == "__main__":
     try:
-        download(url, filename)
+        download(url, file_name)
         setup(file_name)
     except Exception as e:
         print(f"Error {e}")