Skip to content

Commit

Permalink
Merge branch 'master' of ssh://github.com/okfn/ckan
Browse files Browse the repository at this point in the history
  • Loading branch information
David Read committed Jan 27, 2012
2 parents 7a51a16 + fda5bbb commit 7297e0b
Show file tree
Hide file tree
Showing 20 changed files with 2,890 additions and 63 deletions.
16 changes: 14 additions & 2 deletions ckan/lib/cli.py
View file
Expand Up @@ -68,8 +68,9 @@ class ManageDb(CkanCommand):
db version # returns current version of data schema
db dump {file-path} # dump to a pg_dump file
db dump-rdf {dataset-name} {file-path}
db simple-dump-csv {file-path}
db simple-dump-json {file-path}
db simple-dump-csv {file-path} # dump just datasets in CSV format
db simple-dump-json {file-path} # dump just datasets in JSON format
db user-dump-csv {file-path} # dump user information to a CSV file
db send-rdf {talis-store} {username} {password}
db load {file-path} # load a pg_dump from a file
db load-only {file-path} # load a pg_dump from a file but don\'t do
Expand Down Expand Up @@ -115,6 +116,8 @@ def command(self):
self.simple_dump_json()
elif cmd == 'dump-rdf':
self.dump_rdf()
elif cmd == 'user-dump-csv':
self.user_dump_csv()
elif cmd == 'create-from-model':
model.repo.create_db()
if self.verbose:
Expand Down Expand Up @@ -238,6 +241,15 @@ def dump_rdf(self):
f.write(rdf)
f.close()

def user_dump_csv(self):
if len(self.args) < 2:
print 'Need csv file path'
return
dump_filepath = self.args[1]
import ckan.lib.dumper as dumper
dump_file = open(dump_filepath, 'w')
dumper.UserDumper().dump(dump_file)

def send_rdf(self):
if len(self.args) < 4:
print 'Need all arguments: {talis-store} {username} {password}'
Expand Down
28 changes: 25 additions & 3 deletions ckan/lib/dumper.py
View file
Expand Up @@ -4,7 +4,7 @@

import ckan.model as model
import ckan.model
from helpers import json
from helpers import json, OrderedDict

class SimpleDumper(object):
'''Dumps just package data but including tags, groups, license text etc'''
Expand Down Expand Up @@ -40,7 +40,7 @@ def dump_csv(self, dump_file_obj, query):
pkg_dict[name_] = value_
del pkg_dict[name]
row_dicts.append(pkg_dict)
writer = PackagesCsvWriter(row_dicts)
writer = CsvWriter(row_dicts)
writer.save(dump_file_obj)

def dump_json(self, dump_file_obj, query):
Expand Down Expand Up @@ -201,7 +201,7 @@ def migrate_06_to_07(self):
values={'name': record.name})
update.execute()

class PackagesCsvWriter:
class CsvWriter:
def __init__(self, package_dict_list=None):
self._rows = []
self._col_titles = []
Expand Down Expand Up @@ -301,3 +301,25 @@ def pkg_to_xl_dict(pkg):
dict_[key_] = value_
del dict_[key]
return dict_

class UserDumper(object):
def dump(self, dump_file_obj):
query = model.Session.query(model.User)
query = query.order_by(model.User.created.asc())

columns = (('id', 'name', 'openid', 'fullname', 'email', 'created', 'about'))
row_dicts = []
for user in query:
row = OrderedDict()
for col in columns:
value = getattr(user, col)
if not value:
value = ''
if col == 'created':
value = str(value) # or maybe dd/mm/yyyy?
row[col] = value
row_dicts.append(row)

writer = CsvWriter(row_dicts)
writer.save(dump_file_obj)
dump_file_obj.close()
7 changes: 4 additions & 3 deletions ckan/lib/navl/dictization_functions.py
View file
Expand Up @@ -60,9 +60,10 @@ def flatten_schema(schema, flattened=None, key=None):
return flattened

def get_all_key_combinations(data, flattented_schema):
'''compare the schema agaist the given data and get all valid tuples that
match the schema ignoring the last value in the tuple.'''
'''Compare the schema against the given data and get all valid tuples that
match the schema ignoring the last value in the tuple.
'''
schema_prefixes = set([key[:-1] for key in flattented_schema])
combinations = set([()])

Expand Down Expand Up @@ -206,7 +207,7 @@ def _remove_blank_keys(schema):
return schema

def validate(data, schema, context=None):
'''validate an unflattened nested dict agiast a schema'''
'''Validate an unflattened nested dict against a schema.'''

context = context or {}

Expand Down
2 changes: 1 addition & 1 deletion ckan/lib/search/sql.py
View file
Expand Up @@ -24,7 +24,7 @@ def run(self, query):
def makelike(field):
_attr = getattr(model.Package, field)
return _attr.ilike('%' + term + '%')
if q and not (q == '""' or q == "''"):
if q and not (q == '""' or q == "''" or q == '*:*'):
terms = q.split()
# TODO: tags ...?
fields = ['name', 'title', 'notes']
Expand Down
2 changes: 1 addition & 1 deletion ckan/logic/schema.py
View file
Expand Up @@ -39,7 +39,7 @@ def default_resource_schema():

schema = {
'id': [ignore_empty, unicode],
'revistion_id': [ignore_missing, unicode],
'revision_id': [ignore_missing, unicode],
'resource_group_id': [ignore],
'package_id': [ignore],
'url': [ignore_empty, unicode],#, URL(add_http=False)],
Expand Down
2 changes: 1 addition & 1 deletion ckan/migration/versions/034_resource_group_table.py
View file
Expand Up @@ -110,7 +110,7 @@ def upgrade(migrate_engine):
# do data transfer
# give resource group a hashed version of package uuid
# so that we can use the same hash calculation on
# the resource and resource revistion table
# the resource and resource revision table
migrate_engine.execute('''
insert into resource_group
select
Expand Down
5 changes: 4 additions & 1 deletion ckan/public/css/style.css
View file
Expand Up @@ -963,6 +963,9 @@ ul.dataset-edit-nav li a:hover {
.dataset-edit-form .resource-add .fileinfo {
margin: 7px 0;
}
.dataset-edit-form button.dataset-delete {
vertical-align: top;
}


/* ================================ */
Expand Down Expand Up @@ -1160,11 +1163,11 @@ body.package.read h3 {
}
.search-result .extra-links {
float: right;
text-align: right;
}
.search-result .view-more-link {
color: #000;
display: block;
text-align: right;
margin-top: 4px;
padding: 3px 22px 3px 10px;
background: url('/images/icons/arrow-right-16-black.png') no-repeat right;
Expand Down
89 changes: 58 additions & 31 deletions ckan/public/scripts/application.js
View file
Expand Up @@ -37,7 +37,7 @@

var isResourceView = $('body.package.resource_read').length > 0;
if (isResourceView) {
CKANEXT.DATAPREVIEW.setupDataPreview(preload_resource);
CKANEXT.DATAPREVIEW.loadPreviewDialog(preload_resource);
}

var isDatasetNew = $('body.package.new').length > 0;
Expand Down Expand Up @@ -80,6 +80,18 @@
el: $el
});
view.render();

// Set up dataset delete button
var select = $('select.dataset-delete');
select.attr('disabled','disabled');
select.css({opacity: 0.3});
$('button.dataset-delete').click(function(e) {
select.removeAttr('disabled');
select.fadeTo('fast',1.0);
$(e.target).css({opacity:0});
$(e.target).attr('disabled','disabled');
return false;
});
}
var isGroupEdit = $('body.group.edit').length > 0;
if (isGroupEdit) {
Expand Down Expand Up @@ -492,7 +504,13 @@ CKAN.View.DatasetEditForm = Backbone.View.extend({
var boundToUnload = false;
return function() {
if (!boundToUnload) {
CKAN.Utils.flashMessage(CKAN.Strings.youHaveUnsavedChanges,'notice');
var parentDiv = $('<div />').addClass('flash-messages');
var messageDiv = $('<div />').html(CKAN.Strings.youHaveUnsavedChanges).addClass('notice').hide();
parentDiv.append(messageDiv);
$('#unsaved-warning').append(parentDiv);
console.log($('#unsaved-warning'));
messageDiv.show(200);

boundToUnload = true;
window.onbeforeunload = function () {
return CKAN.Strings.youHaveUnsavedChanges;
Expand All @@ -501,7 +519,7 @@ CKAN.View.DatasetEditForm = Backbone.View.extend({
}
}();

$form.find('input').live('change', function(e) {
$form.find('input,select').live('change', function(e) {
$target = $(e.target);
// Entering text in the 'add' box does not represent a change
if ($target.closest('.resource-add').length==0) {
Expand Down Expand Up @@ -749,17 +767,6 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
my.dialogId = 'ckanext-datapreview';
my.$dialog = $('#' + my.dialogId);

// Initialize data explorer on Resource view page
//
// resourceData: resource as simple hash (suitable for initializing backbone model or result of backboneModel.toJSON())
my.setupDataPreview = function(resourceData) {
// initialize the tableviewer system
DATAEXPLORER.TABLEVIEW.initialize(my.dialogId);
resourceData.formatNormalized = my.normalizeFormat(resourceData.format);

my.loadPreviewDialog(resourceData);
};

// **Public: Loads a data preview**
//
// Fetches the preview data object from the link provided and loads the
Expand All @@ -770,16 +777,30 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
//
// Returns nothing.
my.loadPreviewDialog = function(resourceData) {
resourceData.url = my.normalizeUrl(resourceData.url);
my.$dialog.html('<h4>Loading ... <img src="http://assets.okfn.org/images/icons/ajaxload-circle.gif" class="loading-spinner" /></h4>');

function initializeDataExplorer(dataset) {
var dataExplorer = new recline.View.DataExplorer({
el: my.$dialog
, model: dataset
, config: {
readOnly: true
}
});
// will have to refactor if this can get called multiple times
Backbone.history.start();
}

// 4 situations
// a) have a webstore_url
// b) csv or xls (but not webstore)
// c) can be treated as plain text
// d) none of the above but worth iframing (assumption is
// that if we got here (i.e. preview shown) worth doing
// something ...)
resourceData.formatNormalized = my.normalizeFormat(resourceData.format);

resourceData.url = my.normalizeUrl(resourceData.url);
if (resourceData.formatNormalized === '') {
var tmp = resourceData.url.split('/');
tmp = tmp[tmp.length - 1];
Expand All @@ -792,18 +813,21 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
}

if (resourceData.webstore_url) {
var _url = resourceData.webstore_url + '.jsontuples?_limit=500';
my.getResourceDataDirect(_url, function(data) {
DATAEXPLORER.TABLEVIEW.showData(data);
DATAEXPLORER.TABLEVIEW.$dialog.dialog('open');
var backend = new recline.Model.BackendWebstore({
url: resourceData.webstore_url
});
recline.Model.setBackend(backend);
var dataset = backend.getDataset();
initializeDataExplorer(dataset);
}
else if (resourceData.formatNormalized in {'csv': '', 'xls': ''}) {
var _url = my.jsonpdataproxyUrl + '?url=' + resourceData.url + '&type=' + resourceData.formatNormalized;
my.getResourceDataDirect(_url, function(data) {
DATAEXPLORER.TABLEVIEW.showData(data);
DATAEXPLORER.TABLEVIEW.$dialog.dialog('open');
var backend = new recline.Model.BackendDataProxy({
url: resourceData.url
, type: resourceData.formatNormalized
});
recline.Model.setBackend(backend);
var dataset = backend.getDataset();
initializeDataExplorer(dataset);
}
else if (resourceData.formatNormalized in {
'rdf+xml': '',
Expand All @@ -823,7 +847,6 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
var _url = my.jsonpdataproxyUrl + '?type=csv&url=' + resourceData.url;
my.getResourceDataDirect(_url, function(data) {
my.showPlainTextData(data);
DATAEXPLORER.TABLEVIEW.$dialog.dialog('open');
});
}
else if (resourceData.formatNormalized in {'html':'', 'htm':''}
Expand All @@ -840,7 +863,10 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
// Cannot reliably preview this item - with no mimetype/format information,
// can't guarantee it's not a remote binary file such as an executable.
var _msg = $('<p class="error">We are unable to preview this type of resource: ' + resourceData.formatNormalized + '</p>');
my.$dialog.html(_msg);
my.showError({
title: 'Unable to preview'
, message: _msg
});
}
};

Expand Down Expand Up @@ -891,22 +917,23 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
//
// Returns nothing.
my.showPlainTextData = function(data) {
// HACK: have to reach into DATAEXPLORER.TABLEVIEW dialog a lot ...
DATAEXPLORER.TABLEVIEW.setupFullscreenDialog();

if(data.error) {
DATAEXPLORER.TABLEVIEW.showError(data.error);
my.showError(data.error);
} else {
var content = $('<pre></pre>');
for (var i=0; i<data.data.length; i++) {
var row = data.data[i].join(',') + '\n';
content.append(my.escapeHTML(row));
}
DATAEXPLORER.TABLEVIEW.$dialog.dialog(DATAEXPLORER.TABLEVIEW.dialogOptions);
DATAEXPLORER.TABLEVIEW.$dialog.append(content);
my.$dialog.html(content);
}
};

my.showError = function (error) {
var _html = '<strong>' + $.trim(error.title) + '</strong><br />' + $.trim(error.message);
my.$dialog.html(_html);
};

my.normalizeFormat = function(format) {
var out = format.toLowerCase();
out = out.split('/');
Expand Down

0 comments on commit 7297e0b

Please sign in to comment.