infochimps / imw
watch download tarball
public
Forks5Right
Watchers26Right
Description: Infinite Monkeywrench - A frameworks for collecting, peeling, and sharing delicious bananas of data.
Homepage: http://infinitemonkeywrench.org
Clone URL: git://github.com/infochimps/imw.git
imw / meta / Makefile
100644 106 lines (83 sloc) 2.999 kb
raw blame history 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

######################################################################
#
# Makefile for infochimps data
#
######################################################################
# utils
map             = $(foreach a,$(2),$(call $(1),$(a)))
 
SHELL           = /bin/bash
 
# wehre to put stuff
ICSDIR          = $(HOME)/ics
SITEDIR         = $(ICSDIR)/site
CODEDIR         = $(ICSDIR)/code
DATADIR         = $(ICSDIR)/data
LOADDIR         = $(DATADIR)/load
FIXD            = ./fixd
PKGD            = ./pkgd
 
ICS_PACK        = rake -f $(CODEDIR)/tools/bin/ics_pack.rb
# Creates the database load files
ICS_DBLOAD	= $(CODEDIR)/tools/bin/ics_dbload.rb
DB_YAML_DUMP    = $(LOADDIR)/tree/ics_schema_dump.yaml
 
 
# Get all schemata in the fixd/ tree
#                                   c scCoDS
FIXD_SCHEMATA   = $(wildcard $(FIXD)/*/*/*/*.icss.yaml)
# Strip off all but the cat/subcat/coll/dataset
DATASETS        = $(patsubst $(FIXD)/%.icss.yaml, %, $(FIXD_SCHEMATA))
PKGD_DIRS       = $(patsubst %, $(PKGD)/%,$(DATASETS))
PKGD_SCHEMATA   = $(join $(PKGD_DIRS:=/), $(notdir $(DATASETS:=.icss.yaml)))
PKGD_COLLS	= $(wildcard $(PKGD)/*/*/*)
 
# # For each dataset, construct a package target
# define PKGDSCH_template
#   # dump onto our list of packages to make
#   PKGD_SCHEMATA += $(PKGD)/$(1)/$(notdir $(1)).icss.yaml
#   # define variables specific to making _this_ package
#   $(PKGD)/$(1)/$(notdir $(1)).icss.yaml: SCHEMA:=$(FIXD)/$(1).icss.yaml
# endef
# $(foreach dataset,$(DATASETS),$(eval $(call PKGDSCH_template,$(dataset))))
 
 
.PHONY: all
all:			$(PKGD_DIRS)
 
 
# $(PKGD_SCHEMATA): 	$(SCHEMA)
# 	$(ICS_PACK) icss=$(abspath $(SCHEMA))
 
$(PKGD_DIRS): $(PKGD)/% : $(FIXD)/%.icss.yaml
	$(ICS_PACK) icss=$<
	touch $@
 
 
# $(DB_LOAD):		$(PKGD_DIRS)
# 	@echo $(ICS_DBLOAD)
 
#	KLUDGE -- working around "argument list too long" errors
$(DB_YAML_DUMP):	$(PKGD_DIRS)
	echo "--- "          >  $(DB_YAML_DUMP)
	echo "dumping all schemata into one bigass file"
	echo $(SHELL)
	for coll_dir in $(PKGD_COLLS) ; do cd $(DATADIR)/$$coll_dir ; cat */*.icss.yaml >> $(DB_YAML_DUMP) ; done ; true
 
.PHONY: db
db:	$(DB_YAML_DUMP)
	$(ICS_DBLOAD)
 
.PHONY: migrate
migrate: db
	cd $(SITEDIR) ; rake db:migrate VERSION=10 ; rake db:migrate VERSION=99
 
.PHONY: bigmigrate
bigmigrate: db
	cd $(SITEDIR) ; rake db:migrate:reset
 
.PHONY: clean
clean: 
        #            c ScCoDsFile
	rm -rf   $(PKGD)/*/*/*/*
 
######################################################################
#
# debug -- Dump helpful info
#
#######################################################################
 
.PHONY: debug
debug:
#	$(info $(FIXD_SCHEMATA))
	$(info $(PKGD_SCHEMATA))
#	$(info $(PKGD_DIRS))
#	$(info $(DATASETS))
#	 $(info $(SCHEMA))
 
 
# CRUFT:
# 
# map             = $(foreach a,$(2),$(call $(1),$(a)))
# fixdsch_dataset = $(patsubst $(FIXD)/%.icss.yaml, %, $(1))
# pkgdsch_dataset = $(patsubst $(PKGD)/%.icss.yaml, %, $(1))
# dataset_pkgdsch = $(PKGD)/$(1)/$(notdir $(1)).icss.yaml
# dataset_fixdsch = $(FIXD)/$(dir $(1)).icss.yaml