Skip to content

Commit

Permalink
Added a --test option to scraper.py to run code tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
Sasha B committed Feb 14, 2012
1 parent d79405b commit b49add1
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 80 deletions.
6 changes: 3 additions & 3 deletions Problem.py
Expand Up @@ -64,9 +64,9 @@ def %s:
"""

## test icons ##
CHECK_MARK = u'\u2713'
CROSS_MARK = u'\u2717'
STOP_MARK = u'\u25A0'
CHECK_MARK = 'Y' # u'\u2713'
CROSS_MARK = 'N' # u'\u2717'
STOP_MARK = 'X' # u'\u25A0'

class Problem(object, IterableUserDict):
"""The class for all TopCoder problems.
Expand Down
2 changes: 1 addition & 1 deletion ProblemFolder.py
Expand Up @@ -134,7 +134,7 @@ def test_problems(self, problems):
for rel_path, number, name in problems:
# load problem
problem = Problem(rel_path + os.sep + (JSON_FILE_FORMAT % name))
print " * Running tests for problem %d: %s * " % (problem[P_PROBLEM_NUMBER], problem[P_PROBLEM_NAME])
print " * Running tests for problem %d: %s *" % (problem[P_PROBLEM_NUMBER], problem[P_PROBLEM_NAME])

# execute python file text
python_filename = rel_path + os.sep + (PYTHON_FILE_FORMAT % problem[P_PROBLEM_DEFINITION]['class'])
Expand Down
5 changes: 4 additions & 1 deletion README
Expand Up @@ -43,4 +43,7 @@ $ scraper.py 10 --smart
$ scraper.py 10 --smart --force

6) Fetch all problems between 10 and 50 on the TopCoder listing, overwriting everything:
$ scraper.py 10,50 --smart --force
$ scraper.py 10,50 --smart --force

7) Test problems 8, 9 and 10
$ scraper.py 8-10 --test
45 changes: 0 additions & 45 deletions marker.py

This file was deleted.

69 changes: 39 additions & 30 deletions scraper.py
Expand Up @@ -55,41 +55,50 @@ def get_topcoder_problem_ids(opener, n, end = None):
help="If specified, operates in smart mode, and looks for n new problems from TopCoder. Specify two numbers to find all problems between these two numbers. Does NOT download existing problems.")
parser.add_argument('-f', '--force', action="store_true",
help="If specified, overwrites existing problem Python files, or (in smart mode) downloads problems even if they already exist.")
parser.add_argument('-t', '--test', action="store_true",
help="If specified, runs tests for the specified problem numbers instead of running them.")
args = parser.parse_args()

# parse out problem numbers
problem_numbers = get_numbers_from_list(args.problems)

# create output directory
# scan output directory
folder = ProblemFolder(args.output_dir)

# connect to TopCoder
print "Connecting to TopCoder...",
opener = connect_to_topcoder()
print "OK"

# are we in smart mode?
if args.smart:
# look for problem numbers first
print "Looking for problems...",
if len(problem_numbers) == 2:
new_ids = get_topcoder_problem_ids(opener, problem_numbers[0], problem_numbers[1])
else:
new_ids = get_topcoder_problem_ids(opener, problem_numbers[0])
print "%d problems found." % len(new_ids)

# don't re-download existing problems, unless in forced mode
if args.force:
problem_numbers = new_ids
else:
existing_ids = folder.get_problem_numbers()
problem_numbers = [x for x in new_ids if x not in existing_ids]
if args.test:
# test specified problems
for n in problem_numbers:
folder.test_problems(folder.find_problem(number = n))

print "--- Scraping %d problems ---" % len(problem_numbers)

# scrape problems
for n in problem_numbers:
print " * Scraping problem %d." % n
folder.scrape_and_add_problem(n, opener=opener, force=args.force)

print "--- OK ---"
else:
# connect to TopCoder
print "Connecting to TopCoder...",
opener = connect_to_topcoder()
print "OK"

# are we in smart mode?
if args.smart:
# look for problem numbers first
print "Looking for problems...",
if len(problem_numbers) == 2:
new_ids = get_topcoder_problem_ids(opener, problem_numbers[0], problem_numbers[1])
else:
new_ids = get_topcoder_problem_ids(opener, problem_numbers[0])
print "%d problems found." % len(new_ids)

# don't re-download existing problems, unless in forced mode
if args.force:
problem_numbers = new_ids
else:
existing_ids = folder.get_problem_numbers()
problem_numbers = [x for x in new_ids if x not in existing_ids]

print "--- Scraping %d problems ---" % len(problem_numbers)

# scrape problems
for n in problem_numbers:
print " * Scraping problem %d." % n
folder.scrape_and_add_problem(n, opener=opener, force=args.force)

print "--- OK ---"

0 comments on commit b49add1

Please sign in to comment.