Skip to content

Commit

Permalink
Additional fixes to slashcode-dearchve:
Browse files Browse the repository at this point in the history
	- Removes backticks from detected nicknames (API has problems
	with these).

	- Fixes problems where newlines appeared in the extracted nickname.
	(along with the extra HTML). This caused problems with nicknames
	that may have been in the database, but weren't getting found.

Makefile:
	Fixed minor syntax errors in changes for RPM support.

Makefile.NEW:
	Proposed new makefile which replaces "#!/usr/bin/perl" with
	whatever is appropriate on the installed system. This file is
	in alpha state, but please test it out on your systems if you
	get the chance:

		make -f Makefile.NEW INIT=... GROUP=... PERL=....
  • Loading branch information
Xliff committed Feb 28, 2001
1 parent faa1e55 commit 83f8033
Show file tree
Hide file tree
Showing 3 changed files with 226 additions and 29 deletions.
28 changes: 14 additions & 14 deletions Makefile
Expand Up @@ -37,20 +37,20 @@ CP = cp
# I should also grab an install-sh instead of using $(CP)
slash:
if ! [ $(RPM) ] ; then \
(cd Slash; $(PERL) Makefile.PL; make) \
(cd Slash; $(PERL) Makefile.PL; make); \
else \
(cd Slash; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \
(cd Slash; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
fi

plugins:
if ! [ $(RPM) ] ; then \
(cd plugins/Search; $(PERL) Makefile.PL; make) \
(cd plugins/Journal; $(PERL) Makefile.PL; make) \
(cd plugins/Ladybug; $(PERL) Makefile.PL; make) \
(cd plugins/Search; $(PERL) Makefile.PL; make); \
(cd plugins/Journal; $(PERL) Makefile.PL; make); \
(cd plugins/Ladybug; $(PERL) Makefile.PL; make); \
else \
(cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \
(cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \
(cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \
(cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
(cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
(cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
fi

all: install
Expand All @@ -63,13 +63,13 @@ install: slash plugins
(cd Slash; make install)
# Lets go install the plugin's libraries
if ! [ $(RPM) ] ; then \
(cd plugins/Search; $(PERL) Makefile.PL; make install) \
(cd plugins/Journal; $(PERL) Makefile.PL; make install) \
(cd plugins/Ladybug; $(PERL) Makefile.PL; make install) \
(cd plugins/Search; $(PERL) Makefile.PL; make install); \
(cd plugins/Journal; $(PERL) Makefile.PL; make install); \
(cd plugins/Ladybug; $(PERL) Makefile.PL; make install); \
else \
(cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install) \
(cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install) \
(cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install) \
(cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \
(cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \
(cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \
fi

# First we do the default sutff
Expand Down
185 changes: 185 additions & 0 deletions Makefile.NEW
@@ -0,0 +1,185 @@
# This code is a part of Slash, and is released under the GPL.
# Copyright 1997-2001 by Open Source Development Network. See README
# and COPYING for more information, or see http://slashcode.com/.
# $Id$

##
## Makefile -- Current one for Slash
##

# the used tools
VERSION = 1.1.4-bender
DISTNAME = slash
DISTVNAME = $(DISTNAME)-$(VERSION)

SHELL = /bin/sh
PERL = perl
NOOP = $(SHELL) -c true
RM_RF = rm -rf
RM = rm -f
SUFFIX = .gz
COMPRESS = gzip --best
TAR = tar
TARFLAGS = cvf
PREOP = @$(NOOP)
POSTOP = @$(NOOP)
TO_UNIX = @$(NOOP)
PREFIX = /usr/local/slash
INIT = /etc
USER = nobody
GROUP = nobody
CP = cp

# Perl scripts, grouped by directory.
BINFILES = `find bin -name CVS -prune -o -type f -print`
SBINFILES = `find sbin -name CVS -prune -o -type f -print`
THEMEFILES = `find themes -name CVS -prune -o -name \*.pl -print`
PLUGINFILES = `find plugins -name CVS -prune -o -name \*.pl -print`

# install the shared object file into Apache
# We should run a script on the binaries to get the right
# version of perl.
# I should also grab an install-sh instead of using $(CP)
slash:
if ! [ $(RPM) ] ; then \
(cd Slash; $(PERL) Makefile.PL; make); \
else \
(cd Slash; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
fi

plugins:
if ! [ $(RPM) ] ; then \
(cd plugins/Search; $(PERL) Makefile.PL; make); \
(cd plugins/Journal; $(PERL) Makefile.PL; make); \
(cd plugins/Ladybug; $(PERL) Makefile.PL; make); \
else \
(cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
(cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
(cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \
fi

all: install

install: slash plugins
# Need to toss in a script here that will fix prefix so
# that if someone wants to install in a different
# directory it will be easy
# Lets go install the libraries
(cd Slash; make install)
# Lets go install the plugin's libraries
if ! [ $(RPM) ] ; then \
(cd plugins/Search; $(PERL) Makefile.PL; make install); \
(cd plugins/Journal; $(PERL) Makefile.PL; make install); \
(cd plugins/Ladybug; $(PERL) Makefile.PL; make install); \
else \
(cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \
(cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \
(cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \
fi

# Create all necessary directories.
install -d $(PREFIX)/bin/ $(PREFIX)/sbin $(PREFIX)/sql/ $(PREFIX)/sql/mysql/ $(PREFIX)/sql/postgresql $(PREFIX)/themes/ $(PREFIX)/themes/slashcode/htdocs/ $(PREFIX)/themes/slashcode/sql/ $(PREFIX)/themes/slashcode/sql/postgresql $(PREFIX)/themes/slashcode/sql/mysql $(PREFIX)/themes/slashcode/backup $(PREFIX)/themes/slashcode/logs/ $(PREFIX)/plugins/ $(PREFIX)/httpd/

# Insure we use the proper perl interpreter in all scripts that we install.
# Note the use of Perl as opposed to dirname(1) and basename(1) which
# may or may not exist on any given system.
# <jamie> I would suggest, Cliff, instead of using sed to write temp files to /tmp, use perl -i.bak -pe, and delete the .bak files immediately afterwards.
# <jamie> Writing files with predictable names to /tmp and then assuming they still contain the same data 140 microseconds later is a small but nonzero security risk...
(if [ "$(PERL)" != "perl" ]; then \
replacewith=`$(PERL) -MConfig -e 'print $$Config{startperl}' | sed -e 's/\([.?/*]\)/\\\\\1/g'`; \
echo "REPLACEWITH=$$replacewith"; \
for f in $(BINFILES) $(SBINFILES) $(THEMEFILES) $(PLUGINFILES); do \
b=`echo $$f | $(PERL) -MFile::Basename -e 'print basename(<STDIN>)'`; \
d=`echo $$f | $(PERL) -MFile::Basename -e 'print dirname(<STDIN>)'`; \
cat $$f | sed -e "s|#!/usr/bin/perl|$$replacewith|" > \
/tmp/$$b.temp; \
install -D /tmp/$$b.temp $(PREFIX)/$$d/$$b; \
rm /tmp/$$b.temp; \
echo "Installing $$f in $(PREFIX)/$$d"; \
done; \
fi)

#install -D bin/install-slashsite bin/install-plugin bin/tailslash bin/template-tool $(PREFIX)/bin/
#install -D sbin/slashd sbin/portald sbin/moderatord sbin/dailyStuff $(PREFIX)/sbin/
cp sql/mysql/slashschema_create.sql $(PREFIX)/sql/mysql/schema.sql
cp sql/postgresql/slashschema_create.sql $(PREFIX)/sql/postgresql/schema.sql
if [ -f $(PREFIX)/httpd/slash.conf ]; then\
echo "Preserving old slash.conf"; \
else \
$(CP) httpd/slash.conf $(PREFIX)/httpd/slash.conf; \
fi

$(CP) httpd/slash.conf $(PREFIX)/httpd/slash.conf.def

# Note the use of -u in the copy commands below. We don't want to overwrite
# any of the perl scritps we've already modified and put into place!

# Now for the default theme (be nice when this goes in themes)
cp -ruv plugins/* $(PREFIX)/plugins/
# Now all other themes
cp -ruv themes/* $(PREFIX)/themes

# this needs BSD support (and Solaris)
if [ -d /etc/init.d ]; then\
install utils/slashd /etc/init.d/; \
ln -s -f ../init.d/slashd $(INIT)/rc3.d/S99slashd; \
ln -s -f ../init.d/slashd $(INIT)/rc6.d/K99slashd; \
elif [ -d /etc/init.d/rc.d/]; then \
install utils/slashd /etc/rc.d/init.d/ \
ln -s -f ../init.d/slashd /etc/rc.d/rc3.d/S99slashd; \
ln -s -f ../init.d/slashd /etc/rc.d/rc6.d/K99slashd; \
else \
echo "This is either BSD or some other OS I do not understand"; \
echo "You will need to look at how to install utils/slashd"; \
fi

touch $(PREFIX)/slash.sites
chown $(USER):$(GROUP) $(PREFIX)
chown -R $(USER):$(GROUP) $(PREFIX)/themes
chown -R $(USER):$(GROUP) $(PREFIX)/sbin
chown -R $(USER):$(GROUP) $(PREFIX)/bin
chown -R $(USER):$(GROUP) $(PREFIX)/sql
chown -R $(USER):$(GROUP) $(PREFIX)/plugins
# Add a @ to suppress output of the echo's
@echo "+--------------------------------------------------------+"; \
echo "| All done. |"; \
echo "| If you want to let Slash handle your httpd.conf file |"; \
echo "| go add: |"; \
echo "| |"; \
echo "| Include $(PREFIX)/httpd/slash.conf |"; \
echo "| |"; \
echo "| to your httpd.conf for apache. |"; \
echo "| If not, cat its content into your httpd.conf file. |"; \
echo "| |"; \
echo "| Thanks for installing Slash. |"; \
echo "+--------------------------------------------------------+"; \


reload: install
apachectl stop
apachectl start
# cleanup
clean:

dist: $(DISTVNAME).tar$(SUFFIX)

$(DISTVNAME).tar$(SUFFIX) : distdir
$(PREOP)
$(TO_UNIX)
$(TAR) $(TARFLAGS) $(DISTVNAME).tar $(DISTVNAME)
$(RM_RF) $(DISTVNAME)
$(COMPRESS) $(DISTVNAME).tar
$(POSTOP)

distdir :
$(RM_RF) $(DISTVNAME)
$(PERL) -MExtUtils::Manifest=manicopy,maniread \
-e "manicopy(maniread(),'$(DISTVNAME)', '$(DIST_CP)');"

manifest :
(cd Slash; make distclean)
$(PERL) -MExtUtils::Manifest -e 'ExtUtils::Manifest::mkmanifest'

rpm :
rpm -ba slash.spec

42 changes: 27 additions & 15 deletions utils/slashcode-dearchive
Expand Up @@ -35,7 +35,7 @@ my %idextractor_regexps = (

# Parameter validation.
my(%opts);
getopts('d:s:f:y:e:n:CX', \%opts);
getopts('d:s:f:y:e:n:CXO', \%opts);
&usage if !keys(%opts);
# Optionally use single parameter as virtual username, however specification
# with -d <virtualuser> will override anything given in @ARGV.
Expand Down Expand Up @@ -122,9 +122,14 @@ if (! $opts{s}) {
}
die "\nCan't find ID using any extractor!\n" if !$sid;
print STDERR "'$sid'\n" if $debug && $sid;
if ($slashdb->sqlSelect('*', 'comments', 'sid=' . quote($sid)) && !$opts{O}) {
print "Data already exists for article '$sid'!!\n";
exit 1;
if ($slashdb->sqlSelect('*', 'comments', 'sid=' . quote($sid))) {
if (!$opts{O}) {
print "Data already exists for article '$sid'!!\n";
exit 1;
} else {
print "Deleting all existing comments from article '$sid'\n";
$slashdb->sqlDo('DELETE FROM comments WHERE sid=' . quote($sid));
}
}
print "Beginning extraction for article '$sid'...\n";

Expand Down Expand Up @@ -158,15 +163,22 @@ while ($bigscalar =~ /$en_regexp/sg) {
$comment,
$sig) = ($1, $2, $3, $4, $5, $6, $7, $8, $9);

print STDERR "EXTRACTED: $1, $2, $3, $4, $5, $6, $7, $8, $9\n" if $debug;
next if !$cid;

print STDERR "EXTRACTED: $1, $2, $3, $4, $5, $6, $7, $8, $9\n"
if $debug;

# Copy $username.
$uname = $username;
# Fix $uname if it is a link.
$uname =~ s/<A HREF=".*?">(.+?)<\/A>/$1/;
# Also extract and remove email information.
$uname =~ s/ <B><FONT.+?>\((.+?)\)<\/FONT><\/B>// and
# Extract and remove email information.
if ($uname =~ s/<B><FONT.+?>\((.+?)\)<\/FONT><\/B>//) {
$email_addy = $1;
}
# Fix $uname if it is a link.
$uname =~ s/<A HREF=".*?">(.+?)<\/A>.+$/$1/s;
# Remove any and all backtics (`) from the user name since this breaks
# the API for some reason.
$uname =~ s/\`//g;

# Grab score from indicator.
if ($score_ind =~ /Score:(\d+)/) {
Expand Down Expand Up @@ -211,7 +223,7 @@ while ($bigscalar =~ /$en_regexp/sg) {
# year. So the simple solution is to strip the DoW which appears
# at the beginning of the string. This really depends on the site
# but this is the default for Slashcode and also works on Slashdot.
$ts =~ s/(?:Mon|Tues|Wednes|Thurs|Fri|Sat|Sun)day\s+//;
$ts =~ s/(?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day\s+//;

# Now grab the zone associated with the date.
$ts =~ /\s([^\ ]+?)$/;
Expand All @@ -234,8 +246,7 @@ while ($bigscalar =~ /$en_regexp/sg) {

# Attribute comment to the proper nickname in the comment if the
# user doesn't turn up in the database.
$comment = "Posted by $username:<BR><BR>$comment"
if (! $uid);
$comment = "Posted by $username:<BR><BR>$comment" if (! $uid);

print "\n", header("Extracted ${\(length($x_data))} bytes"),
($cid ? "#$cid\n" : ""),
Expand All @@ -256,7 +267,7 @@ while ($bigscalar =~ /$en_regexp/sg) {

# Assign data to hash.
$data{$cid}->{subject} = $subj;
$data{$cid}->{'-uid'} = ($uid) ? $uid : -1;
$data{$cid}->{'-uid'} = $uid;
$data{$cid}->{date} = $gmt_ts;
$data{$cid}->{comment} = $comment;
$data{$cid}->{'-points'} = ($score) ? $score : 0;
Expand All @@ -280,8 +291,9 @@ $bigscalar = '';
# Now write data to database.
print "Writing to database...";
my $nr = 0;
foreach (sort { $data{$a}->{ts} cmp $data{$b}->{ts} } keys(%data)) {
# We process comments into the database serially by DATE.
foreach (keys(%data)) {
# We process comments into the database by CID and let it worry about
# all of the sorting.

# Copy and sanitize data for insert into the comments table.
my(%sqldata) = %{$data{$_}};
Expand Down

0 comments on commit 83f8033

Please sign in to comment.