Skip to content

Commit

Permalink
Merge pull request #55 from brownhead/get_archive
Browse files Browse the repository at this point in the history 
Finished fixing up get_archive api call.
  • Loading branch information
@brownhead2
brownhead2 committed Nov 3, 2012
2 parents 01ac9cf + c986c5d commit d12cd3f
Show file tree
Hide file tree
Showing 6 changed files with 325 additions and 144 deletions.
52 changes: 52 additions & 0 deletions docs/src/galah.api/commands/get_archive.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
get_archive
===========

Downloads all the submissions for a given assignment as a tarball. The tarball
is structures as follows:

.. code-block:: none
user@school.edu
|---- 2012-10-20-23-44-55
| |---- main.cpp
| +---- something_else.cpp
|---- 2012-10-20-23-33-21
+ +---- main.cpp
other_user@schoole.edu
|---- 2012-10-19-21-33-43
+ +---- main.cpp
So the top level directories of the tarball will all be names of users, and
inside of each of the user directories there will be directories named by the
timestamp of the submission (following the format
``YEAR-MONTH-DAY-HOUR-MINUTE-SECOND``. Finally, inside of the timestamped
directories are the actual user submissions.

In the unlikely scenario that two submissions by the same user have the same
timestamp, the timestamped directory will follow the format
``YEAR-MONTH-DAY-HOUR-MINUTE-SECOND-RANDOMNUMBER``.

Reference
---------

.. function:: get_archie(assignment[, email = ""])

:param assignment: The exact id of the assignment.
:param email: You can optionally specify a user's email to filter on, and
only that user's submissions will be downloaded.

Example Usage
-------------

>>> get_archive 5091654355c448096df90687
--Logged in as jsull003@ucr.edu--
Your archive is being created. You have 0 jobs ahead of you.
The server is requesting that you download a file...
Where would you like to save it (default: ./submissions.tar.gz)?: submissions-all.tar.gz
File saved to submissions-all.tar.gz.
>>> get_archive 5091654355c448096df90687 student@ucr.edu
--Logged in as jsull003@ucr.edu--
Your archive is being created. You have 0 jobs ahead of you.
The server is requesting that you download a file...
Where would you like to save it (default: ./submissions.tar.gz)?:
File saved to ./submissions.tar.gz.
4 changes: 3 additions & 1 deletion docs/src/galah.api/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,6 @@ Below is reference material for every API command Galah supports.

commands/delete_assignment

commands/assignment_info
commands/assignment_info

commands/get_archive
90 changes: 75 additions & 15 deletions galah/api/api_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@

# The default configuration settings
config = {
"api_url": "http://localhost:5000/api/call",
"login_url": "http://localhost:5000/api/login"
"galah_host": "http://localhost:5000",
}

import json
Expand Down Expand Up @@ -37,11 +36,17 @@ def _form_call(api_name, *args, **kwargs):

# We'll need to store any cookies the server gives us (mainly the auth cookie)
# and requests' sessions give us a nice way to do that.
_session = requests.session()
session = requests.session()

def login(email, password):
request = _session.post(
config["login_url"], data = {"email": email, "password": password}
"""
Attempts to authenticate with Galah using the given credentials.
"""

request = session.post(
config["galah_host"] + "/api/login",
data = {"email": email, "password": password}
)

request.raise_for_status()
Expand All @@ -53,16 +58,22 @@ def login(email, password):
# Nothing bad happened, go ahead and return what the server sent back
return request.text

def call(api_name, *args, **kwargs):
import time
def call(interactive, api_name, *args, **kwargs):
"""
Makes an API call to the server with the given arguments. This function will
block until the server sends its response.
Iff interactive is True then call will take care of printing to the console
itself, and will prompt the user if the server wants to push any downloads
down, None is returned. Otherwise, pushes will be ignored and the text sent
from the server will be returned, nothing will be printed to the console.
"""

# May throw a requests.ConnectionError here if galah.api is unavailable.
request = _session.post(
config["api_url"],
request = session.post(
config["galah_host"] + "/api/call",
data = _to_json(_form_call(api_name, *args, **kwargs)),
headers = {"Content-Type": "application/json"}
)
Expand All @@ -85,9 +96,58 @@ def call(api_name, *args, **kwargs):
# because of some issues with Flask, so we have this custom header.
if request.headers["X-CallSuccess"] != "True":
raise RuntimeError(request.text)

# If we're not in interactive mode, our job is done already.
if not interactive:
return request.text

print request.text

# Check if the server wants us to download a file
if "X-Download" in request.headers:
default_name = request.headers.get(
"X-Download-DefaultName", "downloaded_file"
)

print "The server is requesting that you download a file..."
save_to = raw_input(
"Where would you like to save it (default: ./%s)?: " % default_name
)

# If they don't type anything in, go with the default.
if not save_to:
save_to = "./" + default_name

if os.path.isfile(save_to):
confirmation = raw_input(
"File %s already exists, would you like to overwrite it "
"(y, n)? " % save_to
)

if not confirmation.startswith("y"):
exit("Aborting.")

# Actually grab the file from the server
while True:
file_request = session.get(
config["galah_host"] + "/" + request.headers["X-Download"]
)

if file_request.status_code == requests.codes.ok:
break

print "Download not ready yet, waiting for server... Retrying " \
"in 2 seconds..."

time.sleep(2)

with open(save_to, "wb") as download_file:
download_file.write(file_request.content)

print "File saved to %s." % save_to

# Nothing bad happened, go ahead and return what the server sent back
return request.text
return None

import sys
from optparse import OptionParser, make_option
Expand Down Expand Up @@ -141,7 +201,7 @@ def exec_to_shell():
script_location = os.path.abspath(script_location)

# Retrieve all of the available commands from the server
api_info = json.loads(call("get_api_info"))
api_info = json.loads(call(False, "get_api_info"))
commands = [i["name"] for i in api_info]

rcfile_path = os.path.join(os.environ["HOME"], ".galah/tmp/shellrc")
Expand Down Expand Up @@ -223,7 +283,7 @@ def exec_to_shell():
print "--Logged in as %s--" % user
except requests.exceptions.ConnectionError as e:
print >> sys.stderr, "Could not connect with the given url '%s':" \
% config["login_url"]
% config["galah_host"]
print >> sys.stderr, "\t" + str(e)

exit(1)
Expand All @@ -235,14 +295,14 @@ def exec_to_shell():
else:
print "--Not logged in--"

# This will fail because duckface is not a proper email, but you should get
# past the authentication...
try:
try:
print call(*args)
# This function actually outputs the result of the call to the
# console.
call(True, *args)
except requests.exceptions.ConnectionError as e:
print >> sys.stderr, "Could not connect with the given url '%s':" \
% config["api_url"]
% config["galah_host"]
print >> sys.stderr, "\t" + str(e)

exit(1)
Expand Down
149 changes: 23 additions & 126 deletions galah/api/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ def __call__(self, current_user, *args, **kwargs):

# Only pass the current user to the function if the function wants it
if len(self.argspec[0]) != 0 and self.argspec[0][0] == "current_user":
return str(self.wrapped_function(current_user, *args, **kwargs))
return self.wrapped_function(current_user, *args, **kwargs)
else:
return str(self.wrapped_function(*args, **kwargs))
return self.wrapped_function(*args, **kwargs)

from decorator import decorator

Expand Down Expand Up @@ -154,7 +154,7 @@ def _to_datetime(time):
## Below are the actual API calls ##
@_api_call()
def get_api_info():
# This function should be memoized
# TODO: This function should be memoized

api_info = []
for k, v in api_calls.items():
Expand Down Expand Up @@ -548,135 +548,32 @@ def delete_assignment(current_user, id):

to_delete.delete()

return "Success! %s deleted." % _assignment_to_str(to_delete)

import threading
import Queue
tar_tasks_queue = Queue.Queue()
tar_tasks_thread = None

# Copied from web.views._upload_submission.SUBMISSION_DIRECTORY. Adding a new
# submission should be transformed into an API call and _upload_submissions
# should use that API call, but this will work for now.
SUBMISSION_DIRECTORY = "/var/local/galah.web/submissions/"

import tempfile
import os
import subprocess
def tar_tasks():
# The thread that executes this function should execute as a daemon,
# therefore there is no reason to allow an explicit exit. It will be
# brutally killed once the app exits.
while True:
# Block until we get a new task.
task = tar_tasks_queue.get()

# Find any expired archives and remove them
# TODO: Remove the actual archives as well.
Archive.objects(expires__lt = datetime.datetime.today()).delete()

# Create a temporary directory we will create our archive in
temp_directory = tempfile.mkdtemp()

# We're going to create a list of file we need to put in the archive
files = [os.path.join(temp_directory, "meta.json")]

# Serialize the meta data and throw it into a file
json.dump(task[1], open(files[0], "w"))

for i in task[1]["submissions"]:
sym_path = os.path.join(temp_directory, i["id"])
os.symlink(os.path.join(SUBMISSION_DIRECTORY, i["id"]), sym_path)
files.append(sym_path)



archive_file = tempfile.mkstemp(suffix = ".tar.gz")[1]

# Run tar and do the actual archiving. Will block until it's finished.
return_code = subprocess.call(
[
"tar", "--dereference", "--create", "--gzip", "--directory",
temp_directory, "--file", archive_file
] + [os.path.relpath(i, temp_directory) for i in files]
)

# Make the results available in the database
archive = Archive.objects.get(id = ObjectId(task[0]))
if return_code != 0:
archive.error_string = \
"tar failed with error code %d." % return_code
else:
archive.file_location = archive_file

archive.expires = \
datetime.datetime.today() + datetime.timedelta(hours = 2)

archive.save()

return "Success! %s deleted." % _assignment_to_str(to_delete)

import tar_tasks
@_api_call(("admin", "teacher"))
def get_submissions(current_user, assignment, email = None):
"""Creates an archive of students' submissions that a teacher or admin
can download.
:param assignment: The assignment that the retrieved submissions will be
for.
:param email: The user that the retrieved submissions will be created by.
If none, all user's who submitted for the given assignment
will be retrieved.
def get_archive(current_user, assignment, email = ""):
the_assignment = _get_assignment(assignment).id

"""

query = {"assignment": ObjectId(assignment)}

# If we were to always add user to the query, then mongo would search for
# documents with email == None in the case that email equals None, which is
# not desirable.
if email:
query["email"] = email

submissions = list(Submission.objects(marked_for_grading = True, **query))

if not submissions:
return "No submissions found."

# Form meta data on each submission that we will soon convert to JSON and
# put inside of the archive we will send the user.
submissions_meta = [
{"id": str(i.id), "user": i.user, "timestamp": str(i.timestamp)}
for i in submissions
]

meta_data = {
"query": {"assignment": assignment, "email": email},
"submissions": submissions_meta
}

# Create a new entry in the database so we can track the progress of the
# job.
new_archive = Archive(requester = current_user.email)
new_archive.save(force_insert = True)

# Determine how many jobs are ahead of this one before we put it in the
# queue.
current_jobs = tar_tasks_queue.qsize()
jobs_ahead = tar_tasks.queue_size()

# We will not perform the work of archiving right now but will instead pass
# if off to another thread to take care of it.
tar_tasks_queue.put((new_archive.id, meta_data))

# If the thread responsible for archiving is not running, start it up.
global tar_tasks_thread
if tar_tasks_thread is None or not tar_tasks_thread.is_alive():
tar_tasks_thread = threading.Thread(name = "tar_tasks", target = tar_tasks)
tar_tasks_thread.start()

return ("Creating archive with id [%s]. Approximately %d jobs ahead of "
"you. Access your archive by trying "
"[Galah Domain]/archives/%s"
% (str(new_archive.id), current_jobs, str(new_archive.id)))

new_task = tar_tasks.Task(
id = ObjectId(),
requester = current_user.email,
assignment = the_assignment,
email = email
)
tar_tasks.add_task(new_task)

return (
"Your archive is being created. You have %d jobs ahead of you." %
jobs_ahead,
{
"X-Download": "archives/" + str(new_task.id),
"X-Download-DefaultName": "submissions.tar.gz"}
)

from types import FunctionType
api_calls = dict((k, v) for k, v in globals().items() if isinstance(v, APICall))
Expand Down

0 comments on commit d12cd3f

Please sign in to comment.