Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Working parsers, and some basic documentation.

  • Loading branch information...
commit 8e1a15e8d4c34d0258f5418f93be24827cf1a7b2 1 parent 38be901
@gtaylor authored
View
22 LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) 2012 Gregory Taylor
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
View
31 README.rst
@@ -1,7 +1,26 @@
-EVE Market Monster
-==================
+Thundercuddles
+==============
-OM NOM NOM NOM
+:Author: Greg Taylor
+:License: BSD
+
+This project is a proof-of-concept for a super-scalable, cheap way to parse
+a large amount of market data from EVE Online players, then make said data
+available via a basic API.
+
+There are already a number of other great sites that do this well, so we have
+to stick to the following principles to make this a worthwhile experiment:
+
+* The application must be able to accept an extremely large number of incoming
+ market orders without performance degrading noticeably.
+* It must be very easy to scale the system.
+* The data must be served in a similar, super-scalable-and-zippy manner.
+
+Current Status
+--------------
+
+This project is in early, early development. Unless you are very masochistic,
+you probably won't derive any value from it just yet.
Installation
------------
@@ -9,3 +28,9 @@ Installation
* pip install -r requirements.txt
* ???
* profit
+
+License
+-------
+
+This project, and all contributed code, are licensed under the BSD License.
+A copy of the BSD License may be found in the repository.
View
3  requirements.txt
@@ -1,2 +1,3 @@
bottle
-simplejson
+simplejson
+gevent
View
5 src/core/README.rst
@@ -0,0 +1,5 @@
+Core
+====
+
+Anything in here is generally useful throughout the entire codebase, and isn't
+tied to any single component.
View
0  src/core/__init__.py
No changes.
View
98 src/core/market_data.py
@@ -0,0 +1,98 @@
+"""
+Data structures for representing market data.
+"""
+from string import Template
+
+# Some order type defines. Use these constants instead of "buy" and "sell"
+# where possible.
+ORDER_TYPE_BUY = "buy"
+ORDER_TYPE_SELL = "sell"
+ORDER_TYPES = [ORDER_TYPE_BUY, ORDER_TYPE_SELL]
+
+class InvalidMarketOrderDataError(Exception):
+ """
+ Raise this when invalid market order data is passed into MarketOrder's
+ constructor.
+ """
+ pass
+
+
+class MarketOrder(object):
+ """
+ Represents a market buy or sell order.
+ """
+
+ def __init__(self, order_id, order_type, region_id, solar_system_id,
+ station_id, type_id,
+ price, volume_entered, volume_remaining, minimum_volume,
+ order_issue_date, order_duration, order_range):
+ """
+ :param int order_id: The unique order ID for this order.
+ :param str order_type: One of 'buy' or 'sell'.
+ :param int region_id: The region the order is in.
+ :param int solar_system_id: The solar system the order is in.
+ :param int station_id: The station the order is in.
+ :param int type_id: The item type of the order.
+ :param float price: The buy/sell price per item.
+ :param int volume_entered: The original amount of the buy/sell order.
+ :param int volume_remaining: The quantity remaining in the order.
+ :param int minimum_volume: The minimum volume that may remain
+ before the order is removed.
+ :param datetime order_issue_date: The time at which the order was
+ first posted.
+ :param int order_duration: The duration (in days) of the order.
+ :param int order_range: No idea what this is.
+ """
+ self.order_id = order_id
+
+ if order_type not in ORDER_TYPES:
+ raise InvalidMarketOrderDataError("Invalid order type.")
+
+ self.order_type = order_type
+ self.region_id = region_id
+ self.solar_system_id = solar_system_id
+ self.station_id = station_id
+ self.type_id = type_id
+ self.price = price
+ self.volume_entered = volume_entered
+ self.volume_remaining = volume_remaining
+ self.minimum_volume = minimum_volume
+ self.order_issue_date = order_issue_date
+ self.order_duration = order_duration
+ self.order_range = order_range
+
+ def __str__(self):
+ """
+ Basic string representation of the order.
+ """
+ template = Template(
+ "Market Order: \n"
+ " order_id: $order_id\n"
+ " order_type: $order_type\n"
+ " region_id: $region_id\n"
+ " solar_system_id: $solar_system_id\n"
+ " station_id: $station_id\n"
+ " type_id: $type_id\n"
+ " price: $price\n"
+ " volume_entered: $volume_entered\n"
+ " volume_remaining: $volume_remaining\n"
+ " minimum_volume: $minimum_volume\n"
+ " order_issue_date: $order_issue_date\n"
+ " order_duration: $order_duration\n"
+ " order_range: $order_range\n"
+ )
+ return template.substitute(
+ order_id = self.order_id,
+ order_type = self.order_type,
+ region_id = self.region_id,
+ solar_system_id = self.solar_system_id,
+ station_id = self.station_id,
+ type_id = self.type_id,
+ price = self.price,
+ volume_entered = self.volume_entered,
+ volume_remaining = self.volume_remaining,
+ minimum_volume = self.minimum_volume,
+ order_issue_date = self.order_issue_date,
+ order_duration = self.order_duration,
+ order_range = self.order_range,
+ )
View
10 src/daemons/gateway/README.rst
@@ -0,0 +1,10 @@
+Gateway WSGI Application
+========================
+
+The gateway application is what the various market data uploaders toss their
+data at. It parses whatever custom format they're using into our standard
+Python representation of a market order (src.core.market_data.MarketOrder), and
+serializes it to JSON, to be shoved into Amazon's Simple Queue Service (SQS).
+
+From there, a worker daemon separate from this one picks it up and does magical
+things to it.
View
4 src/daemons/gateway/parsers/README.rst
@@ -0,0 +1,4 @@
+Uploader Parsers
+================
+
+This module contains parsers for the various market data uploading formats.
View
2  src/daemons/gateway/parsers/__init__.py
@@ -0,0 +1,2 @@
+#noinspection PyUnresolvedReferences
+from src.daemons.gateway.parsers import eve_marketeer
View
70 src/daemons/gateway/parsers/eve_marketeer.py
@@ -0,0 +1,70 @@
+"""
+Parser for the EVE Marketeer and EVE Market Data uploader format.
+"""
+import csv
+import logging
+import datetime
+from StringIO import StringIO
+from src.core.market_data import MarketOrder, ORDER_TYPE_BUY, ORDER_TYPE_SELL
+
+logger = logging.getLogger(__name__)
+
+def parse_from_request(request):
+ """
+ Given a request, parse the contents and return a generator of
+ :py:class:`src.core.market_data.MarketOrder` instances. Each instance
+ represents a market order.
+ """
+ #print "FORMS", request.forms.items()
+
+ # This is a CSV string that has line breaks as line separators.
+ log = request.forms.log
+ # Stuff the string here so the csv reader module can pull from it.
+ # TODO: Look at getting the csv reader to read the string directly.
+ log_buf = StringIO(log)
+
+ upload_type = request.forms.upload_type
+ if upload_type != 'orders':
+ # This isn't an orders upload, we want no part in it.
+ logger.error("Upload type other than 'order' found. Yuck.")
+ return
+
+ # The item type ID.
+ type_id = request.forms.type_id
+ region_id = request.forms.region_id
+ # Parse the market log buffer as a CSV.
+ for row in csv.reader(log_buf, delimiter=','):
+ order_id,\
+ order_type,\
+ solar_system_id,\
+ station_id,\
+ price,\
+ volume_entered,\
+ volume_remaining,\
+ minimum_volume,\
+ order_issue_date,\
+ order_duration,\
+ order_range = row
+
+ # Now we cast each bit of data as a poor man's validator.
+ order_id = int(order_id)
+ order_type = ORDER_TYPE_BUY if order_type == "b" else ORDER_TYPE_SELL
+ solar_system_id = int(solar_system_id)
+ station_id = int(station_id)
+ price = float(price)
+ volume_entered = int(volume_entered)
+ volume_remaining = int(float(volume_remaining))
+ minimum_volume = int(minimum_volume)
+ order_issue_date = datetime.datetime.strptime(
+ order_issue_date, "%Y-%m-%d %H:%M:%S")
+ order_duration = int(order_duration)
+ order_range = int(order_range)
+
+ # Finally, instantiate and pop out a MarketOrder instance, which will
+ # be re-serialized in our standard format and sent to SQS for the
+ # workers to pull and save.
+ yield MarketOrder(
+ order_id, order_type, region_id, solar_system_id, station_id,
+ type_id, price, volume_entered, volume_remaining, minimum_volume,
+ order_issue_date, order_duration, order_range,
+ )
View
50 src/daemons/gateway/wsgi.py
@@ -1,27 +1,31 @@
-import csv
-from StringIO import StringIO
-from bottle import route, run, request, post
+"""
+This WSGI application accepts market data uploads from various uploader clients.
+The various URLs below are structured to pass off the parsing based on what
+format the data is in.
-@post('/api/upload/')
-def index():
- print "FORMS", request.forms.items()
+The parsed representation of the order is then sent off to Amazon Simple
+Queue Service (SQS), where the worker processes can pull them from for
+processing.
+"""
+from gevent import monkey; monkey.patch_all()
+from bottle import route, run, request, post
- log = request.forms.log
- log_buf = StringIO(log)
- for row in csv.reader(log_buf, delimiter=','):
- order_id, \
- buy_sell, \
- solar_system_id, \
- station_id, \
- price, \
- vol_entered,\
- vol_remaining, \
- min_volume, \
- order_issuedate, \
- order_duration, \
- order_range = row
+from src.daemons.gateway import parsers
- upload_type = request.forms.upload_type
- region_id = request.forms.region_id
+@post('/api/upload/')
+def upload_eve_marketeer():
+ """
+ This view accepts uploads in EVE Marketeer or EVE Marketdata format. These
+ typically arrive via the EVE Unified Uploader client.
+ """
+ order_generator = parsers.eve_marketeer.parse_from_request(request)
+ for order in order_generator:
+ print order
-run(host='localhost', port=8080)
+# Start the built-in Bottle server for development, for now.
+run(
+ host='localhost',
+ port=8080,
+ server='gevent',
+ reloader=True
+)
Please sign in to comment.
Something went wrong with that request. Please try again.