From 83f8033635c4baf5c89659356907093e3d04dd9d Mon Sep 17 00:00:00 2001 From: Cliff Wood Date: Wed, 28 Feb 2001 18:35:34 +0000 Subject: [PATCH] Additional fixes to slashcode-dearchve: - Removes backticks from detected nicknames (API has problems with these). - Fixes problems where newlines appeared in the extracted nickname. (along with the extra HTML). This caused problems with nicknames that may have been in the database, but weren't getting found. Makefile: Fixed minor syntax errors in changes for RPM support. Makefile.NEW: Proposed new makefile which replaces "#!/usr/bin/perl" with whatever is appropriate on the installed system. This file is in alpha state, but please test it out on your systems if you get the chance: make -f Makefile.NEW INIT=... GROUP=... PERL=.... --- Makefile | 28 +++--- Makefile.NEW | 185 ++++++++++++++++++++++++++++++++++++++ utils/slashcode-dearchive | 42 +++++---- 3 files changed, 226 insertions(+), 29 deletions(-) create mode 100644 Makefile.NEW diff --git a/Makefile b/Makefile index 315315273..1cc9f2482 100644 --- a/Makefile +++ b/Makefile @@ -37,20 +37,20 @@ CP = cp # I should also grab an install-sh instead of using $(CP) slash: if ! [ $(RPM) ] ; then \ - (cd Slash; $(PERL) Makefile.PL; make) \ + (cd Slash; $(PERL) Makefile.PL; make); \ else \ - (cd Slash; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \ + (cd Slash; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ fi plugins: if ! [ $(RPM) ] ; then \ - (cd plugins/Search; $(PERL) Makefile.PL; make) \ - (cd plugins/Journal; $(PERL) Makefile.PL; make) \ - (cd plugins/Ladybug; $(PERL) Makefile.PL; make) \ + (cd plugins/Search; $(PERL) Makefile.PL; make); \ + (cd plugins/Journal; $(PERL) Makefile.PL; make); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL; make); \ else \ - (cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \ - (cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \ - (cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make) \ + (cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ + (cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ fi all: install @@ -63,13 +63,13 @@ install: slash plugins (cd Slash; make install) # Lets go install the plugin's libraries if ! [ $(RPM) ] ; then \ - (cd plugins/Search; $(PERL) Makefile.PL; make install) \ - (cd plugins/Journal; $(PERL) Makefile.PL; make install) \ - (cd plugins/Ladybug; $(PERL) Makefile.PL; make install) \ + (cd plugins/Search; $(PERL) Makefile.PL; make install); \ + (cd plugins/Journal; $(PERL) Makefile.PL; make install); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL; make install); \ else \ - (cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install) \ - (cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install) \ - (cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install) \ + (cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \ + (cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \ fi # First we do the default sutff diff --git a/Makefile.NEW b/Makefile.NEW new file mode 100644 index 000000000..bb9b0b76a --- /dev/null +++ b/Makefile.NEW @@ -0,0 +1,185 @@ +# This code is a part of Slash, and is released under the GPL. +# Copyright 1997-2001 by Open Source Development Network. See README +# and COPYING for more information, or see http://slashcode.com/. +# $Id$ + +## +## Makefile -- Current one for Slash +## + +# the used tools +VERSION = 1.1.4-bender +DISTNAME = slash +DISTVNAME = $(DISTNAME)-$(VERSION) + +SHELL = /bin/sh +PERL = perl +NOOP = $(SHELL) -c true +RM_RF = rm -rf +RM = rm -f +SUFFIX = .gz +COMPRESS = gzip --best +TAR = tar +TARFLAGS = cvf +PREOP = @$(NOOP) +POSTOP = @$(NOOP) +TO_UNIX = @$(NOOP) +PREFIX = /usr/local/slash +INIT = /etc +USER = nobody +GROUP = nobody +CP = cp + +# Perl scripts, grouped by directory. +BINFILES = `find bin -name CVS -prune -o -type f -print` +SBINFILES = `find sbin -name CVS -prune -o -type f -print` +THEMEFILES = `find themes -name CVS -prune -o -name \*.pl -print` +PLUGINFILES = `find plugins -name CVS -prune -o -name \*.pl -print` + +# install the shared object file into Apache +# We should run a script on the binaries to get the right +# version of perl. +# I should also grab an install-sh instead of using $(CP) +slash: + if ! [ $(RPM) ] ; then \ + (cd Slash; $(PERL) Makefile.PL; make); \ + else \ + (cd Slash; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ + fi + +plugins: + if ! [ $(RPM) ] ; then \ + (cd plugins/Search; $(PERL) Makefile.PL; make); \ + (cd plugins/Journal; $(PERL) Makefile.PL; make); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL; make); \ + else \ + (cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ + (cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make); \ + fi + +all: install + +install: slash plugins +# Need to toss in a script here that will fix prefix so +# that if someone wants to install in a different +# directory it will be easy + # Lets go install the libraries + (cd Slash; make install) + # Lets go install the plugin's libraries + if ! [ $(RPM) ] ; then \ + (cd plugins/Search; $(PERL) Makefile.PL; make install); \ + (cd plugins/Journal; $(PERL) Makefile.PL; make install); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL; make install); \ + else \ + (cd plugins/Search; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \ + (cd plugins/Journal; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \ + (cd plugins/Ladybug; $(PERL) Makefile.PL INSTALLSITEARCH=/var/tmp/slash-buildroot/usr/local/lib/perl/5.6.0 INSTALLSITELIB=/var/tmp/slash-buildroot/usr/local/share/perl/5.6.0; make install); \ + fi + + # Create all necessary directories. + install -d $(PREFIX)/bin/ $(PREFIX)/sbin $(PREFIX)/sql/ $(PREFIX)/sql/mysql/ $(PREFIX)/sql/postgresql $(PREFIX)/themes/ $(PREFIX)/themes/slashcode/htdocs/ $(PREFIX)/themes/slashcode/sql/ $(PREFIX)/themes/slashcode/sql/postgresql $(PREFIX)/themes/slashcode/sql/mysql $(PREFIX)/themes/slashcode/backup $(PREFIX)/themes/slashcode/logs/ $(PREFIX)/plugins/ $(PREFIX)/httpd/ + + # Insure we use the proper perl interpreter in all scripts that we install. + # Note the use of Perl as opposed to dirname(1) and basename(1) which + # may or may not exist on any given system. + # I would suggest, Cliff, instead of using sed to write temp files to /tmp, use perl -i.bak -pe, and delete the .bak files immediately afterwards. + # Writing files with predictable names to /tmp and then assuming they still contain the same data 140 microseconds later is a small but nonzero security risk... + (if [ "$(PERL)" != "perl" ]; then \ + replacewith=`$(PERL) -MConfig -e 'print $$Config{startperl}' | sed -e 's/\([.?/*]\)/\\\\\1/g'`; \ + echo "REPLACEWITH=$$replacewith"; \ + for f in $(BINFILES) $(SBINFILES) $(THEMEFILES) $(PLUGINFILES); do \ + b=`echo $$f | $(PERL) -MFile::Basename -e 'print basename()'`; \ + d=`echo $$f | $(PERL) -MFile::Basename -e 'print dirname()'`; \ + cat $$f | sed -e "s|#!/usr/bin/perl|$$replacewith|" > \ + /tmp/$$b.temp; \ + install -D /tmp/$$b.temp $(PREFIX)/$$d/$$b; \ + rm /tmp/$$b.temp; \ + echo "Installing $$f in $(PREFIX)/$$d"; \ + done; \ + fi) + + #install -D bin/install-slashsite bin/install-plugin bin/tailslash bin/template-tool $(PREFIX)/bin/ + #install -D sbin/slashd sbin/portald sbin/moderatord sbin/dailyStuff $(PREFIX)/sbin/ + cp sql/mysql/slashschema_create.sql $(PREFIX)/sql/mysql/schema.sql + cp sql/postgresql/slashschema_create.sql $(PREFIX)/sql/postgresql/schema.sql + if [ -f $(PREFIX)/httpd/slash.conf ]; then\ + echo "Preserving old slash.conf"; \ + else \ + $(CP) httpd/slash.conf $(PREFIX)/httpd/slash.conf; \ + fi + + $(CP) httpd/slash.conf $(PREFIX)/httpd/slash.conf.def + + # Note the use of -u in the copy commands below. We don't want to overwrite + # any of the perl scritps we've already modified and put into place! + + # Now for the default theme (be nice when this goes in themes) + cp -ruv plugins/* $(PREFIX)/plugins/ + # Now all other themes + cp -ruv themes/* $(PREFIX)/themes + + # this needs BSD support (and Solaris) + if [ -d /etc/init.d ]; then\ + install utils/slashd /etc/init.d/; \ + ln -s -f ../init.d/slashd $(INIT)/rc3.d/S99slashd; \ + ln -s -f ../init.d/slashd $(INIT)/rc6.d/K99slashd; \ + elif [ -d /etc/init.d/rc.d/]; then \ + install utils/slashd /etc/rc.d/init.d/ \ + ln -s -f ../init.d/slashd /etc/rc.d/rc3.d/S99slashd; \ + ln -s -f ../init.d/slashd /etc/rc.d/rc6.d/K99slashd; \ + else \ + echo "This is either BSD or some other OS I do not understand"; \ + echo "You will need to look at how to install utils/slashd"; \ + fi + + touch $(PREFIX)/slash.sites + chown $(USER):$(GROUP) $(PREFIX) + chown -R $(USER):$(GROUP) $(PREFIX)/themes + chown -R $(USER):$(GROUP) $(PREFIX)/sbin + chown -R $(USER):$(GROUP) $(PREFIX)/bin + chown -R $(USER):$(GROUP) $(PREFIX)/sql + chown -R $(USER):$(GROUP) $(PREFIX)/plugins +# Add a @ to suppress output of the echo's + @echo "+--------------------------------------------------------+"; \ + echo "| All done. |"; \ + echo "| If you want to let Slash handle your httpd.conf file |"; \ + echo "| go add: |"; \ + echo "| |"; \ + echo "| Include $(PREFIX)/httpd/slash.conf |"; \ + echo "| |"; \ + echo "| to your httpd.conf for apache. |"; \ + echo "| If not, cat its content into your httpd.conf file. |"; \ + echo "| |"; \ + echo "| Thanks for installing Slash. |"; \ + echo "+--------------------------------------------------------+"; \ + + +reload: install + apachectl stop + apachectl start +# cleanup +clean: + +dist: $(DISTVNAME).tar$(SUFFIX) + +$(DISTVNAME).tar$(SUFFIX) : distdir + $(PREOP) + $(TO_UNIX) + $(TAR) $(TARFLAGS) $(DISTVNAME).tar $(DISTVNAME) + $(RM_RF) $(DISTVNAME) + $(COMPRESS) $(DISTVNAME).tar + $(POSTOP) + +distdir : + $(RM_RF) $(DISTVNAME) + $(PERL) -MExtUtils::Manifest=manicopy,maniread \ + -e "manicopy(maniread(),'$(DISTVNAME)', '$(DIST_CP)');" + +manifest : + (cd Slash; make distclean) + $(PERL) -MExtUtils::Manifest -e 'ExtUtils::Manifest::mkmanifest' + +rpm : + rpm -ba slash.spec + diff --git a/utils/slashcode-dearchive b/utils/slashcode-dearchive index d856fb467..82e3af437 100755 --- a/utils/slashcode-dearchive +++ b/utils/slashcode-dearchive @@ -35,7 +35,7 @@ my %idextractor_regexps = ( # Parameter validation. my(%opts); -getopts('d:s:f:y:e:n:CX', \%opts); +getopts('d:s:f:y:e:n:CXO', \%opts); &usage if !keys(%opts); # Optionally use single parameter as virtual username, however specification # with -d will override anything given in @ARGV. @@ -122,9 +122,14 @@ if (! $opts{s}) { } die "\nCan't find ID using any extractor!\n" if !$sid; print STDERR "'$sid'\n" if $debug && $sid; -if ($slashdb->sqlSelect('*', 'comments', 'sid=' . quote($sid)) && !$opts{O}) { - print "Data already exists for article '$sid'!!\n"; - exit 1; +if ($slashdb->sqlSelect('*', 'comments', 'sid=' . quote($sid))) { + if (!$opts{O}) { + print "Data already exists for article '$sid'!!\n"; + exit 1; + } else { + print "Deleting all existing comments from article '$sid'\n"; + $slashdb->sqlDo('DELETE FROM comments WHERE sid=' . quote($sid)); + } } print "Beginning extraction for article '$sid'...\n"; @@ -158,15 +163,22 @@ while ($bigscalar =~ /$en_regexp/sg) { $comment, $sig) = ($1, $2, $3, $4, $5, $6, $7, $8, $9); - print STDERR "EXTRACTED: $1, $2, $3, $4, $5, $6, $7, $8, $9\n" if $debug; + next if !$cid; + + print STDERR "EXTRACTED: $1, $2, $3, $4, $5, $6, $7, $8, $9\n" + if $debug; # Copy $username. $uname = $username; - # Fix $uname if it is a link. - $uname =~ s/(.+?)<\/A>/$1/; - # Also extract and remove email information. - $uname =~ s/ \((.+?)\)<\/FONT><\/B>// and + # Extract and remove email information. + if ($uname =~ s/\((.+?)\)<\/FONT><\/B>//) { $email_addy = $1; + } + # Fix $uname if it is a link. + $uname =~ s/(.+?)<\/A>.+$/$1/s; + # Remove any and all backtics (`) from the user name since this breaks + # the API for some reason. + $uname =~ s/\`//g; # Grab score from indicator. if ($score_ind =~ /Score:(\d+)/) { @@ -211,7 +223,7 @@ while ($bigscalar =~ /$en_regexp/sg) { # year. So the simple solution is to strip the DoW which appears # at the beginning of the string. This really depends on the site # but this is the default for Slashcode and also works on Slashdot. - $ts =~ s/(?:Mon|Tues|Wednes|Thurs|Fri|Sat|Sun)day\s+//; + $ts =~ s/(?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day\s+//; # Now grab the zone associated with the date. $ts =~ /\s([^\ ]+?)$/; @@ -234,8 +246,7 @@ while ($bigscalar =~ /$en_regexp/sg) { # Attribute comment to the proper nickname in the comment if the # user doesn't turn up in the database. - $comment = "Posted by $username:

$comment" - if (! $uid); + $comment = "Posted by $username:

$comment" if (! $uid); print "\n", header("Extracted ${\(length($x_data))} bytes"), ($cid ? "#$cid\n" : ""), @@ -256,7 +267,7 @@ while ($bigscalar =~ /$en_regexp/sg) { # Assign data to hash. $data{$cid}->{subject} = $subj; - $data{$cid}->{'-uid'} = ($uid) ? $uid : -1; + $data{$cid}->{'-uid'} = $uid; $data{$cid}->{date} = $gmt_ts; $data{$cid}->{comment} = $comment; $data{$cid}->{'-points'} = ($score) ? $score : 0; @@ -280,8 +291,9 @@ $bigscalar = ''; # Now write data to database. print "Writing to database..."; my $nr = 0; -foreach (sort { $data{$a}->{ts} cmp $data{$b}->{ts} } keys(%data)) { - # We process comments into the database serially by DATE. +foreach (keys(%data)) { + # We process comments into the database by CID and let it worry about + # all of the sorting. # Copy and sanitize data for insert into the comments table. my(%sqldata) = %{$data{$_}};