Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: studentkittens/webarchive
base: 7aa15c6af0
...
head fork: studentkittens/webarchive
compare: ef10e0c755
Checking mergeability… Don't worry, you can still create the pull request.
  • 10 commits
  • 6 files changed
  • 0 commit comments
  • 1 contributor
View
121 src/python/crawler/git.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# encoding: utf-8
+import subprocess
+import os
+
+__author__ = 'Christopher Pahl'
+
+class Git(object):
+ """A (overly-simple) Wrapper for the git binary"""
+ def __init__(self, domain):
+ """Instance a new Git Wrapper for a certain domain
+
+ :param domain: A domain found in the archive
+ """
+ # TODO: get the path to the domain
+ self.__domain = domain
+ self.__gitdir = os.path.join(self.__domain, '.git')
+ self.__basecmd = 'git --git-dir {git_dir} --work-tree {git_cwd} '.format(
+ git_dir = self.__gitdir,
+ git_cwd = self.__domain)
+
+ def __call(self, command):
+ """
+ Execute a git script
+
+ Prepend the basecmd to each line in command,
+ and subprocess.call() the resulting string synchronously
+
+ :param command: a multiline git script
+ :param forkShell: call subprocess.call with shell=True
+ :returns: the returncode of the last command in line
+ """
+ command = '\n'.join([self.__basecmd + line.strip()
+ for line in command.splitlines()
+ if len(line) > 0])
+
+ print('\n//////////////////\n')
+ print(command)
+ return subprocess.call(command, shell=True)
+
+ def init(self):
+ """
+ Create a new archive at specified domain path
+
+ The target directory does not need to exit yet
+
+ :returns: 0 on success, another rc on failure
+ """
+ return self.__call('init ' + self.__domain)
+
+ def checkout(self, target = 'master'):
+ """
+ checkout a certain point (tag, branch or commit)
+
+ :target: the target to visit
+ :returns: 0 on success, another rc on failure
+ """
+ return self.__call('checkout {}'.format(target))
+
+ def branch(self, branch_name = 'empty'):
+ """
+ create a new named branch
+
+ :branch_name: the name of the new branch, may not exist yet
+ :returns: 0 on success, another rc on failure
+ """
+ return self.__call('checkout -fb {}'.format(branch_name))
+
+ def commit(self, message = 'edit'):
+ """
+ commit any changes made
+
+ git add . and git commit -am <message> is done
+
+ :message: The commit message
+ :returns: 0 on success, another rc on failure
+ """
+ return self.__call("""
+ add {path} &&
+ commit -am '{msg}'""".format(
+ path = self.__domain,
+ msg = message))
+
+
+if __name__ == '__main__':
+ import unittest
+
+ TEST_DIR = '/tmp/git_test/'
+ class GitTest(unittest.TestCase):
+ def setUp(self):
+ self.__repo = Git(TEST_DIR)
+ self.__repo.init()
+
+ def add_file(self, name, mode):
+ with open(os.path.join(TEST_DIR, name), mode) as dummy:
+ dummy.write('Hello Kitteh!')
+
+ def test_commit(self):
+ self.add_file('file', 'w')
+ self.assertEqual(self.__repo.commit(message = 'init'), 0)
+
+ def test_branch(self):
+ # make empty master
+ self.add_file('empty', 'w')
+ self.assertEqual(self.__repo.commit(), 0)
+
+ # branch to a new branch and commit something
+ self.assertEqual(self.__repo.branch('2405TT2323'), 0)
+ self.add_file('file_new', 'w')
+ self.assertEqual(self.__repo.commit(), 0)
+
+ # change back to master
+ self.assertEqual(self.__repo.checkout(), 0)
+ self.add_file('file_new', 'w')
+ self.assertEqual(self.__repo.commit(), 0)
+
+ def tearDown(self):
+ subprocess.call(['rm', '-rf', TEST_DIR])
+ pass
+
+ unittest.main()
View
2  src/python/init/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+# encoding: utf-8
View
7 src/python/init/init.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+__author__ = 'Christopher Pahl'
+
+def init_archive(path):
+ pass
View
204 src/python/javadapter/server.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""
+###########################################################################
+# Javadapter #
+###########################################################################
+
+A simple Server with a textprotocol, representing the API to Java.
+
+Usage: Start with python server.py <port>,
+ test with telnet localhost <port>, or
+ python test_client <port>
+
+ Type 'quit' on the server-cmd to exit
+"""
+
+__author__ = 'Christopher Pahl'
+
+import socketserver
+import threading
+import sys
+
+def lock_handler(*args):
+ """
+ Returns 'true' or 'false'
+ """
+ return 'true\n'
+def try_lock_handler(*args):
+ """
+ Returns 'true' or 'false'
+ """
+ return 'false\n'
+def unlock_handler(*args):
+ """
+ Returns 'true' or 'false'
+ """
+ return 'maybe\n'
+def checkout_handler(*args):
+ """
+ Returns path to domain
+ """
+ return '\n'.join(*args) + '\n'
+def commit_handler(*args):
+ """
+ Returns 'true' or 'false'
+ """
+ return 'sure\n'
+
+# Simple dictionary based description of the protocol
+# command
+# => takes: how many arguments to take
+# => func: a handler that does something with the args
+# and return some bytebuffer
+PROTOCOL = {
+ 'lock': {
+ 'takes': 1,
+ 'func': lock_handler
+ },
+ 'try_lock': {
+ 'takes': 1,
+ 'func': try_lock_handler
+ },
+ 'unlock': {
+ 'takes': 1,
+ 'func': unlock_handler
+ },
+ 'checkout': {
+ 'takes': 2,
+ 'func': checkout_handler
+ },
+ 'commit': {
+ 'takes': 2,
+ 'func': commit_handler
+ }
+ }
+
+class ProtocolError(Exception):
+ """raise a 'ACK cause'
+
+ Simple Exception that offers
+ a sendable bytebuffer for errors
+ via self.failure
+ """
+ def __init__(self, msg):
+ """
+ :param msg: a string message, describing the error
+ """
+ super(ProtocolError, self).__init__(msg)
+ self.failure = b'ACK ' + bytes(msg, 'UTF-8')
+
+class AdapterHandler(socketserver.StreamRequestHandler):
+ """The RequestHandler class for the Javadapter
+
+ It is instantiated once per connection to the server, and must
+ override the handle() method to implement communication to the
+ client. It is never instantiated explicitely.
+ """
+ def serve_request(self):
+ """Handle a certain command
+
+ Try to see a meaning in a command,
+ raise an ProtocolError or return a
+ bytebuffer with a suitable reponse,
+ which is ended with a linefeed in any case
+ """
+ response = b''
+ try:
+ # Try to get a handler for this command,
+ # if not part of the protocol raise a ProtocolError
+ # and send an 'ACK ...' back to the caller
+ handler = PROTOCOL[self.__cmd]
+ if len(self.__arg) != handler['takes']:
+ # More error checking might be done here,
+ # but currently only the number of args is checked
+ raise ProtocolError(
+ '"{cmd}" takes exactly {arg} argument(s)'.format(
+ cmd = self.__cmd,
+ arg = handler['takes']))
+
+ # the handlers simply return strings, but for send()
+ # we need bytes. If func() returns a False value,
+ # an empty string is used as response instead
+ response = bytes(handler['func'](self.__arg) or '', 'UTF-8')
+ except KeyError:
+ raise ProtocolError('Unknown command: ' + self.__cmd)
+ else:
+ return response
+
+ def handle(self):
+ """
+ Implemented from RequestHandler
+ """
+ while True:
+ # Convert input to a list of trimmed strings
+ line = [str(x, 'UTF-8')
+ for x in self.rfile.readline().split()]
+
+ # Got EOF, so we better quit
+ if len(line) == 0:
+ print('Quitting connection "%s" to client' %
+ threading.current_thread().name)
+ return
+
+ if len(line) > 0:
+ self.__cmd = line[0]
+ self.__arg = line[1:]
+
+ # No matter what, something is send always,
+ # even it's a ProtocolError
+ try:
+ response = self.serve_request()
+ send_data = response + b'OK'
+ except ProtocolError as err:
+ send_data = err.failure
+ finally:
+ self.wfile.write(send_data + b'\n')
+
+
+class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+ """
+ A 'own' socketserver, with ThreadingMixIn
+
+ Reference: http://docs.python.org/py3k/library/socketserver.html
+ """
+ pass
+
+def start(host = 'localhost', port = 42421):
+ """
+ Start the Javadapter server, and exit once done
+
+ :param host: the host to start the server on (does anythinh but localhost work?)
+ :param port: the port on which the server listens on
+ :returns: a server, on which shutdown() can be called
+ """
+ # Spawn a new thread for each connection
+ server = ThreadedTCPServer((host, port), AdapterHandler)
+ server_thread = threading.Thread(target=server.serve_forever)
+ server_thread.daemon = True
+ server_thread.start()
+
+ return server
+
+if __name__ == "__main__":
+ def main():
+ """
+ main() for cmd use, pass a port as only arg
+ """
+ if len(sys.argv) < 2:
+ print('usage: {prog} port'.format(prog = sys.argv[0]))
+ sys.exit(-1)
+ try:
+ server = start('localhost', int(sys.argv[1]))
+ while True:
+ server_cmd = input('>>> ').strip()
+ if server_cmd == 'quit':
+ break
+ else:
+ print('Unknown command')
+ server.shutdown()
+ except KeyboardInterrupt:
+ print('Quitting Server because of Ctrl-C')
+ sys.exit(0)
+ main()
View
25 src/python/javadapter/telnet_client.py
@@ -0,0 +1,25 @@
+import socket
+import sys
+
+HOST, PORT = "localhost", int(sys.argv[1])
+
+# Create a socket (SOCK_STREAM means a TCP socket)
+sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+try:
+ sock.connect((HOST, PORT))
+ while True:
+ data = bytes(input('>>> '), 'UTF-8')
+
+ # Connect to server and send data
+ sock.sendall(data + b"\n")
+
+ # Receive data from the server and shut down
+ received = sock.recv(1024)
+
+ print("Sent: {}".format(data))
+ print("Received: {}".format(received))
+except EOFError as err:
+ print('Quitting')
+finally:
+ sock.close()
View
38 src/python/javadapter/test_client.py
@@ -0,0 +1,38 @@
+import unittest
+import socket
+
+class TestJavadapter(unittest.TestCase):
+ def setUp(self):
+ host, port = "localhost", 42424
+ self.__sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.__sock.connect((host,port))
+
+ def tearDown(self):
+ self.__sock.close()
+
+ def dialog(self, message):
+ sent = bytes(message, 'UTF-8') + b'\n'
+ self.__sock.sendall(sent)
+ recv = self.__sock.recv(1024)
+ print('-----------')
+ print('Sent:', sent)
+ print('Recv:', recv)
+
+ return recv
+
+ def test_wrong_command(self):
+ self.assertEqual(self.dialog('bad'),
+ b'ACK Unknown command: bad\n')
+
+ def test_bad_argnum(self):
+ self.assertEqual(self.dialog('lock'),
+ b'ACK "lock" takes exactly 1 argument(s)\n')
+ self.assertEqual(self.dialog('checkout a b c'),
+ b'ACK "checkout" takes exactly 2 argument(s)\n')
+
+ def test_multiline(self):
+ self.assertEqual(self.dialog('checkout heise.de now'),
+ b'heise.de\nnow\nOK\n')
+
+if __name__ == '__main__':
+ unittest.main()

No commit comments for this range

Something went wrong with that request. Please try again.