Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 76 lines (58 sloc) 1.931 kb
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
1 #!/bin/bash
2
d48284c Neil Williams Make mr_top jobs more portable.
spladug authored
3 # expects two environment variables
4 # REDDIT_ROOT = path to the root of the reddit public code; the directory with the Makefile
5 # REDDIT_CONFIG = path to the ini file to use
6
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
7 export USER=ri
8
9 # e.g. link or comment
10 export KIND="$1"
11 # e.g. prec01 for links, db02s8 for comments
12 export LINKDBHOST="$2"
d48284c Neil Williams Make mr_top jobs more portable.
spladug authored
13 # e.g. 5432 for default pg or 6543 for pgbouncer
14 export DB_PORT=6543
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
15
16 # e.g. hour
17 export INTERVAL="$3"
18
19 # e.g., '("hour",)'
20 export TIMES="$4"
21
22 export THING=/scratch/profile-thing-$KIND.$INTERVAL.dump
23 export DTHING=/scratch/profile-data-$KIND.$INTERVAL.dump
24
d48284c Neil Williams Make mr_top jobs more portable.
spladug authored
25 cd $REDDIT_ROOT
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
26
27 if [ -e $THING ]; then
28 echo cannot start because $THING exists
29 ls -l $THING
30 exit 1
31 fi
32
33 trap "rm -f $THING $DTHING" SIGINT SIGTERM
34
35 # make this exist immediately to act as a lock
36 touch $THING
37
d48284c Neil Williams Make mr_top jobs more portable.
spladug authored
38 psql -F"\t" -A -t -d newreddit -U $USER -h $LINKDBHOST -p $DB_PORT \
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
39 -c "\\copy (select t.thing_id, 'thing', '$KIND',
40 t.ups, t.downs, t.deleted, t.spam, extract(epoch from t.date)
41 from reddit_thing_$KIND t
42 where not t.deleted
43 and t.date > now() - interval '1 $INTERVAL'
44 )
45 to '$THING'"
46
47 # get the min thing_id
48 MINID=`head -n 1 $THING | awk '{print $1}'`
49
d48284c Neil Williams Make mr_top jobs more portable.
spladug authored
50 psql -F"\t" -A -t -d newreddit -U $USER -h $LINKDBHOST -p $DB_PORT \
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
51 -c "\\copy (select d.thing_id, 'data', '$KIND',
52 d.key, d.value
53 from reddit_data_$KIND d
54 where d.thing_id >= $MINID
55 and d.key = 'author_id'
56 )
57 to '$DTHING'"
58
59 function mrsort {
60 #psort -T/mnt/tmp -S50m
61 sort -T/scratch -S200m
62 }
63
64 function f {
d48284c Neil Williams Make mr_top jobs more portable.
spladug authored
65 paster --plugin=r2 run $REDDIT_CONFIG r2/lib/mr_account.py -c "$1"
a39f0bc david king Update cron scripts to reflect reality.
ketralnis authored
66 }
67
68 cat $THING $DTHING | \
69 mrsort | \
70 f "join_links()" | \
71 f "time_listings($TIMES)" | \
72 mrsort | \
73 f "write_permacache()"
74
75 rm $THING $DTHING
Something went wrong with that request. Please try again.