Merge branch 'dev-community' into john/export-help

LICENSE

@@ -1,3 +1,8 @@
+This license does not apply to any DuckDuckGo logos or marks that may be contained
+in this repo. DuckDuckGo logos and marks are licensed separately under the CCBY-NC-ND 4.0
+license (https://creativecommons.org/licenses/by-nc-nd/4.0/), and official up-to-date
+versions can be downloaded from https://duckduckgo.com/press.
+
 This software is copyright (c) 2011 by DuckDuckGo, Inc. https://duckduckgo.com,
 DuckDuckGo Community
 

dist.ini

@@ -27,6 +27,7 @@ File::ShareDir::Install = 0
 [GatherDir]
 exclude_filename=script/ddgc_deploy_dev.pl
 exclude_filename=script/ddgc_release.sh
+prune_directory=node_modules
 
 [PruneCruft]
 [ManifestSkip]

lib/DDGC/Config.pm

@@ -160,6 +160,7 @@ has_conf asana_personal_access_token => DDGC_ASANA_PERSONAL_ACCESS_TOKEN => unde
 has_conf asana_workspace_id => DDGC_ASANA_WORKSPACE_ID => undef;
 has_conf asana_oncall_project => DDGC_ASANA_ONCALL_PROJECT => undef;
 has_conf asana_oncall_section => DDGC_ASANA_ONCALL_SECTION => undef;
+has_conf asana_security_tag => DDGC_ASANA_SECURITY_TAG => undef;
 has_conf pagerduty_api_key => DDGC_PAGERDUTY_API_KEY => undef;
 
 # DANGER: DEACTIVATES PASSWORD CHECK FOR ALL USERACCOUNTS!!!!!!!!!!!!!!!!!!!!!!

lib/DDGC/DB/Result/Token/Domain/Language.pm

@@ -157,13 +157,16 @@ msgstr ""
 EOF
 	$intro > $po;
 	my %doublecheck;
-	for my $tl ($self->search_related('token_languages',{},{
+	for my $tl ($self->search_related('token_languages',{
+		'token.retired' => 0
+	},{
 		prefetch => ['token',{ 
 			token_domain_language => [qw( language token_domain )],
 			token_language_translations => [{ user => { user_languages => 'language' }, token_language_translation_votes => 'user' }],
 		}],
 		order_by => [ 'token_language_translations.updated', 'token_language_translations.id' ],
 	})->all) {
+		next if $tl->token_domain_language->language->locale eq 'en_US';
 		$tl->auto_use; # should be renamed
 		my $msgid = $tl->token->msgid;
 		$msgid .= '||||msgctxt||||'.$tl->token->msgctxt if $tl->token->msgctxt;

lib/DDGC/DB/Result/Token/Language.pm

@@ -111,6 +111,8 @@ has_many 'token_language_translations', 'DDGC::DB::Result::Token::Language::Tran
 	cascade_delete => 1,
 };
 
+has $_ => ( is => 'rw', default => sub { 0 } ) for qw/ user invalid username user_voted vote_count /;
+
 unique_constraint [qw/ token_id token_domain_language_id /];
 
 sub event_related {
@@ -149,6 +151,8 @@ sub gettext_snippet {
 		$vars{'msgstr'} = $self->gettext_escape($self->msgstr0) if $self->msgstr0;
 	}
 	return unless %vars || $fallback;
+	$vars{notes} = join "\n", map { "#. $_" } grep { $_ } split /[\n\r]+/, $self->token->notes
+		if $self->token->notes;
 	$vars{msgid} = $self->gettext_escape($self->token->msgid);
 	$vars{msgctxt} = $self->gettext_escape($self->token->msgctxt) if $self->token->msgctxt;
 	if ($self->token->msgid_plural) {
@@ -171,9 +175,19 @@ sub gettext_escape {
 	return $content;
 }
 
+sub delete_msgstr {
+	my ( $self, $user ) = @_;
+	return "" unless $user->is('translation_manager');
+	for ( map { "msgstr$_" } 0..5 ) {
+		$self->$_(undef);
+	}
+	$self->update;
+}
+
 sub gettext_snippet_formatter {
 	my ( $self, %vars ) = @_;
 	my $return;
+	$return = ( delete $vars{notes} ) . "\n" if $vars{notes};
 	for (qw( msgctxt msgid msgid_plural )) {
 		$return .= $_.' "'.(delete $vars{$_}).'"'."\n" if $vars{$_};
 	}

lib/DDGC/DB/Result/Token/Language/Translation.pm

@@ -5,14 +5,17 @@ use Moose;
 use MooseX::NonMoose;
 extends 'DDGC::DB::Base::Result';
 use DBIx::Class::Candy;
-
+use Regexp::Common 'profanity';
 use DateTime;
 use Locale::Simple;
 
 use namespace::autoclean;
 
 table 'token_language_translation';
 
+# Yes, this isn't how XSS is handled - we simply want to reduce moderation overhead
+my $html_re = qr/<[^\w<>]*(?:[^<>"'\s]*:)?[^\w<>]*(?:\W*s\W*c\W*r\W*i\W*p\W*t|\W*f\W*o\W*r\W*m|\W*s\W*t\W*y\W*l\W*e|\W*s\W*v\W*g|\W*m\W*a\W*r\W*q\W*u\W*e\W*e|(?:\W*l\W*i\W*n\W*k|\W*o\W*b\W*j\W*e\W*c\W*t|\W*e\W*m\W*b\W*e\W*d|\W*a\W*p\W*p\W*l\W*e\W*t|\W*p\W*a\W*r\W*a\W*m|\W*i?\W*f\W*r\W*a\W*m\W*e|\W*b\W*a\W*s\W*e|\W*b\W*o\W*d\W*y|\W*m\W*e\W*t\W*a|\W*i\W*m\W*a?\W*g\W*e?|\W*v\W*i\W*d\W*e\W*o|\W*a\W*u\W*d\W*i\W*o|\W*b\W*i\W*n\W*d\W*i\W*n\W*g\W*s|\W*s\W*e\W*t|\W*i\W*s\W*i\W*n\W*d\W*e\W*x|\W*a\W*n\W*i\W*m\W*a\W*t\W*e)[^>\w])|(?:<\w[\s\S]*[\s\0\/]|['"])(?:formaction|style|background|src|lowsrc|ping|on(?:d(?:e(?:vice(?:(?:orienta|mo)tion|proximity|found|light)|livery(?:success|error)|activate)|r(?:ag(?:e(?:n(?:ter|d)|xit)|(?:gestur|leav)e|start|drop|over)?|op)|i(?:s(?:c(?:hargingtimechange|onnect(?:ing|ed))|abled)|aling)|ata(?:setc(?:omplete|hanged)|(?:availabl|chang)e|error)|urationchange|ownloading|blclick)|Moz(?:M(?:agnifyGesture(?:Update|Start)?|ouse(?:PixelScroll|Hittest))|S(?:wipeGesture(?:Update|Start|End)?|crolledAreaChanged)|(?:(?:Press)?TapGestur|BeforeResiz)e|EdgeUI(?:C(?:omplet|ancel)|Start)ed|RotateGesture(?:Update|Start)?|A(?:udioAvailable|fterPaint))|c(?:o(?:m(?:p(?:osition(?:update|start|end)|lete)|mand(?:update)?)|n(?:t(?:rolselect|extmenu)|nect(?:ing|ed))|py)|a(?:(?:llschang|ch)ed|nplay(?:through)?|rdstatechange)|h(?:(?:arging(?:time)?ch)?ange|ecking)|(?:fstate|ell)change|u(?:echange|t)|l(?:ick|ose))|m(?:o(?:z(?:pointerlock(?:change|error)|(?:orientation|time)change|fullscreen(?:change|error)|network(?:down|up)load)|use(?:(?:lea|mo)ve|o(?:ver|ut)|enter|wheel|down|up)|ve(?:start|end)?)|essage|ark)|s(?:t(?:a(?:t(?:uschanged|echange)|lled|rt)|k(?:sessione|comma)nd|op)|e(?:ek(?:complete|ing|ed)|(?:lec(?:tstar)?)?t|n(?:ding|t))|u(?:ccess|spend|bmit)|peech(?:start|end)|ound(?:start|end)|croll|how)|b(?:e(?:for(?:e(?:(?:scriptexecu|activa)te|u(?:nload|pdate)|p(?:aste|rint)|c(?:opy|ut)|editfocus)|deactivate)|gin(?:Event)?)|oun(?:dary|ce)|l(?:ocked|ur)|roadcast|usy)|a(?:n(?:imation(?:iteration|start|end)|tennastatechange)|fter(?:(?:scriptexecu|upda)te|print)|udio(?:process|start|end)|d(?:apteradded|dtrack)|ctivate|lerting|bort)|DOM(?:Node(?:Inserted(?:IntoDocument)?|Removed(?:FromDocument)?)|(?:CharacterData|Subtree)Modified|A(?:ttrModified|ctivate)|Focus(?:Out|In)|MouseScroll)|r(?:e(?:s(?:u(?:m(?:ing|e)|lt)|ize|et)|adystatechange|pea(?:tEven)?t|movetrack|trieving|ceived)|ow(?:s(?:inserted|delete)|e(?:nter|xit))|atechange)|p(?:op(?:up(?:hid(?:den|ing)|show(?:ing|n))|state)|a(?:ge(?:hide|show)|(?:st|us)e|int)|ro(?:pertychange|gress)|lay(?:ing)?)|t(?:ouch(?:(?:lea|mo)ve|en(?:ter|d)|cancel|start)|ime(?:update|out)|ransitionend|ext)|u(?:s(?:erproximity|sdreceived)|p(?:gradeneeded|dateready)|n(?:derflow|load))|f(?:o(?:rm(?:change|input)|cus(?:out|in)?)|i(?:lterchange|nish)|ailed)|l(?:o(?:ad(?:e(?:d(?:meta)?data|nd)|start)?|secapture)|evelchange|y)|g(?:amepad(?:(?:dis)?connected|button(?:down|up)|axismove)|et)|e(?:n(?:d(?:Event|ed)?|abled|ter)|rror(?:update)?|mptied|xit)|i(?:cc(?:cardlockerror|infochange)|n(?:coming|valid|put))|o(?:(?:(?:ff|n)lin|bsolet)e|verflow(?:changed)?|pen)|SVG(?:(?:Unl|L)oad|Resize|Scroll|Abort|Error|Zoom)|h(?:e(?:adphoneschange|l[dp])|ashchange|olding)|v(?:o(?:lum|ic)e|ersion)change|w(?:a(?:it|rn)ing|heel)|key(?:press|down|up)|(?:AppComman|Loa)d|no(?:update|match)|Request|zoom))[\s\0]*=/;
+
 sub u { shift->token_language->u }
 
 sub event_related {
@@ -137,6 +140,7 @@ sub votes { shift->token_language_translation_votes(@_) }
 before insert => sub {
 	my ( $self ) = @_;
 	$self->sprintf_check;
+	$self->basic_content_checks;
 };
 
 after insert => sub {
@@ -155,6 +159,7 @@ sub user_voted {
 sub force_check {
 	my ( $self ) = @_;
 	$self->sprintf_check;
+	$self->basic_content_checks;
 	$self->update;
 }
 
@@ -180,6 +185,15 @@ sub sprintf_check {
 	}
 }
 
+sub basic_content_checks {
+	my ( $self ) = @_;
+	my @msgstrs = grep { $_ } map { my $msgstr = "msgstr$_" ; $self->$msgstr } 0..5;
+	for my $msgstr ( @msgstrs ) {
+		$self->check_result(0) if $msgstr =~ /$RE{profanity}/i;
+		$self->check_result(0) if $msgstr =~ /$html_re/gi;
+	}
+}
+
 sub vote_count {
 	my ( $self ) = @_;
 	$self->token_language_translation_votes->count;

lib/DDGC/DB/ResultSet/Token/Language.pm

@@ -10,8 +10,9 @@ sub untranslated {
 	my ( $self, $token_domain_id, $language_id, $scalar_ignore_ids ) = @_;
 	my @ignore_ids = $scalar_ignore_ids ? @{$scalar_ignore_ids} : ();
 	$self->search({
-		'token_domain_id' => $token_domain_id,
+		'token_domain_language.token_domain_id' => $token_domain_id,
 		'language_id' => $language_id,
+		'token.retired' => 0,
 		-and => [
 			'me.id' => { -not_in => \@ignore_ids },
 			-or => [
@@ -26,7 +27,7 @@ sub untranslated {
 	},{
 		join => [ {
 			token_language_translations => 'token_language_translation_votes'
-		}, 'token_domain_language' ],
+		}, 'token_domain_language', 'token' ],
 		order_by => { -asc => ($self->me.'created') },
 	});
 }
@@ -43,10 +44,11 @@ sub untranslated_all {
 		'token_language_translations.id' => undef,
 		'token_domain_language.language_id' => { -in => \@language_ids},
 		'token_domain.active' => 1,
+		'token.retired' => 0,
 		($self->me.'id') => { -not_in => \@ignore_ids },
 		$self->result_source->schema->ddgc->is_live ? ( 'token_domain.key' => { -like => 'duckduckgo-%' } ) : (),
 	},{
-		join => [ 'token_language_translations', { token_domain_language => 'token_domain' } ],
+		join => [ 'token', 'token_language_translations', { token_domain_language => 'token_domain' } ],
 		order_by => { -asc => ($self->me.'created') },
 	});
 }
@@ -65,6 +67,7 @@ sub unvoted_all {
 		'token_language_translations.id' => { -not => undef },
 		'token_domain_language.language_id' => { -in => \@language_ids},
 		'token_domain.active' => 1,
+		'token.retired' => 0,
 		$token_domain_id ? ( 'token_domain_language.token_domain_id' => $token_domain_id ) : (),
 		-and => [
 			($self->me.'id') => { -not_in => \@ignore_ids },
@@ -81,7 +84,7 @@ sub unvoted_all {
 		],
 		$schema->ddgc->is_live ? ( 'token_domain.key' => { -like => 'duckduckgo-%' } ) : (),
 	},{
-		join => [ {
+		join => [ 'token', {
 			token_language_translations => 'token_language_translation_votes'
 		}, { token_domain_language => 'token_domain' } ],
 		order_by => { -desc => ($self->me.'created') },
@@ -103,13 +106,14 @@ sub unvoted {
 	my @ignore_ids = $scalar_ignore_ids ? @{$scalar_ignore_ids} : ();
 	$self->search({
 		'token_language_translations.id' => { -not => undef },
-		'token_domain_id' => $token_domain_id,
+		'token_domain_language.token_domain_id' => $token_domain_id,
 		'language_id' => $language_id,
+		'token.retired' => 0,
 		-and => [
 			($self->me.'id') => { -not_in => \@ignore_ids },
 			($self->me.'id') => { -not_in => $self->search({
 				'token_language_translation_votes.users_id' => $user->id,
-				'token_domain_id' => $token_domain_id,
+				'token_domain_language.token_domain_id' => $token_domain_id,
 				'language_id' => $language_id,
 			},{
 				select => 'user_voted.id',
@@ -120,7 +124,7 @@ sub unvoted {
 			})->as_query},
 		],
 	},{
-		join => [ {
+		join => [ 'token', {
 			token_language_translations => 'token_language_translation_votes'
 		}, 'token_domain_language' ],
 		order_by => { -desc => 'me.created' },

lib/DDGC/LocaleDist.pm

@@ -90,7 +90,7 @@ sub BUILD {
 	my $lib_dir = dir($self->build_dir,'lib',@lib_dir_parts);
 	my $testdir = dir($self->build_dir,'t');
 
-	my $tokencount = $self->token_domain->tokens->search()->count;
+	my $tokencount = $self->token_domain->tokens->search({ 'retired' => 0 })->count;
 	my $domainname = $self->token_domain->name;
 
 	my %locales;

lib/DDGC/Web.pm

@@ -54,12 +54,6 @@ __PACKAGE__->config(
 		enable => $ENV{DDGC_ACTIVATE_ERRORCATCHING}||0,
 		emit_module => 'Catalyst::Plugin::ErrorCatcher::Email',
 	},
-	'Plugin::ErrorCatcher::Email' => {
-		to => $ENV{DDGC_ERROR_EMAIL} // 'ddgc@duckduckgo.com',
-		from => 'noreply@duckduckgo.com',
-		subject => '[DuckDuckGo Community] %p %l CRASH!!!',
-		use_tags => 1,
-	},
 	'Plugin::Static::Simple' => {
 		dirs => [
 			'root'

lib/DDGC/Web/Controller/Feedback.pm

@@ -189,7 +189,8 @@ sub step :Chained('feedback') :PathPart('') :Args(1) {
                project => $c->d->config->asana_oncall_project,
                section => $c->d->config->asana_oncall_section,
            } ],
-           assignee => $c->d->pagerduty->on_call
+           assignee => $c->d->pagerduty->on_call,
+           tags => [ $c->d->config->asana_security_tag ]
          }
        );
     } else {

lib/DDGC/Web/Controller/InstantAnswer.pm

@@ -4,8 +4,7 @@ use Moose;
 use namespace::autoclean;
 use Try::Tiny;
 use Time::Local;
-use JSON;
-use JSON::MaybeXS ':all';
+use JSON::MaybeXS ':legacy';
 use Net::GitHub::V3;
 use DateTime;
 use LWP::UserAgent;
@@ -851,45 +850,11 @@ sub commit_save :Chained('commit_base') :PathPart('save') :Args(0) {
     return $c->forward($c->view('JSON'));
 }
 
-# Install the data on the beta server
-# Data is an hash of arrays
-# {goodies: [#pr1, #pr2, #pr3], spice: [#pr1, #pr2] ... and so on
-sub send_to_beta :Chained('base') :PathPart('send_to_beta') :Args(0) {
-    my ( $self, $c ) = @_;
-    $c->check_action_token;
-
-    my $ua = LWP::UserAgent->new;
-
-    my $result = '';
-    $c->stash->{x}->{result} = $result;
-    return $c->forward($c->view('JSON')) unless ($c->req->params->{data} && $c->user && $c->user->admin);
-
-    my $server = "http://beta.duckduckgo.com/ia_install";
-    my $key = $ENV{'BETA_KEY'};
-    my $decoded_data = from_json($c->req->params->{data});
-
-    for my $data (@{$decoded_data}) {
-        my $req = HTTP::Request->new(GET => $server);
-        my $header_data = "sha1=".Digest::SHA::hmac_sha1_hex(to_json($data), $key);
-
-        $req->header('content-type' => 'application/json');
-        $req->header("x-hub-signature" => $header_data);
-        $req->content(to_json($data));
-
-        my $resp = $ua->request($req);
-
-        $result = $resp->is_success? 1 : 0;
-        $c->stash->{x}->{result} = $result;
-    }
-
-    return $c->forward($c->view('JSON'));
-}
-
 sub beta_req {
     my ($self) = @_;
 
     my $ua = LWP::UserAgent->new;
-    my $server = "http://beta.duckduckgo.com/installed.json";
+    my $server = "https://s3.amazonaws.com/ddg-community/beta/installed.json";
     my $req = HTTP::Request->new(GET => $server);
 
     my $resp;

lib/DDGC/Web/Controller/Translate.pm

@@ -58,6 +58,24 @@ sub logged_in :Chained('base') :PathPart('') :CaptureArgs(0) {
 	}
 }
 
+sub active_tokens :Chained('base') :PathPart('active_tokens.json') :Args(1) {
+	my ( $self, $c, $domain ) = @_;
+	$domain ||= 'duckduckgo-duckduckgo';
+	$c->stash->{x} = {
+		tokens => [
+			map { { id => $_->id, msgid => $_->msgid } }
+			$c->d->rs('Token')
+			->search({
+				retired => 0,
+				'token_domain.key' => $domain,
+			},
+			{ prefetch => 'token_domain' })
+			->all
+		]
+	};
+	$c->forward( $c->view('JSON') );
+}
+
 sub untranslated_all :Chained('logged_in') :Args(0) {
 	my ( $self, $c ) = @_;
 	$c->wiz_start( 'UntranslatedAll' );
@@ -161,6 +179,51 @@ sub translation_check :Chained('translation') :PathPart('check') :Args(1) {
 	$c->forward( $c->view('JSON') );
 }
 
+sub delete_msgstr :Chained('logged_in') :PathPart('delete_live') :Args(1) {
+	my ( $self, $c, $token_language_id ) = @_;
+	my $tl = $c->d->rs('Token::Language')->find({ id => $token_language_id });
+	if ( !$tl ) {
+		$c->response->status(404);
+		return $c->detach;
+	}
+	if ( $tl->delete_msgstr( $c->user ) ) {
+		$c->stash->{x} = {
+			check_result => 2,
+		};
+	} else {
+		$c->response->status(403);
+		$c->stash->{x} = { error => 'Access denied' };
+	}
+	$c->forward( $c->view('JSON') );
+}
+
+sub single_token :Chained('logged_in') CaptureArgs(1) {
+	my ( $self, $c, $token_id ) = @_;
+	$c->stash->{token} = $c->d->rs('Token')->find({ id => $token_id });
+	if (!$c->stash->{token}) {
+		$c->response->status(404);
+		return $c->detach;
+	}
+}
+
+sub token_retire :Chained('single_token') :PathPart('retire') :Args(1) {
+	my ( $self, $c, $retire ) = @_;
+	if ( !$c->user->admin ) {
+		$c->response->status(403);
+		$c->stash->{x} = {
+			error => 403
+		}
+	}
+	else {
+		$c->stash->{token}->retired($retire);
+		$c->stash->{token}->update;
+		$c->stash->{x} = {
+			check_result => $c->stash->{token}->retired,
+		};
+	}
+	$c->forward( $c->view('JSON') );
+}
+
 sub vote :Chained('translation') :CaptureArgs(1) {
 	my ( $self, $c, $vote ) = @_;
 	$c->require_action_token;
@@ -199,12 +262,15 @@ sub domain :Chained('logged_in') :PathPart('') :CaptureArgs(1) {
 						},)->get_column('token_language_id')->as_query,
 					},
 					'undone_count.token_domain_language_id' => { -ident => 'me.id' },
+					'token.retired' => 0,
 				],
 			},{
-				join => 'token_language_translations', alias => 'undone_count'
+				join => [ 'token', 'token_language_translations' ],
+				alias => 'undone_count'
 			})->count_rs->as_query,
 			token_total_count => $c->d->rs('Token')->search({
 				'total_count.token_domain_id' => { -ident => 'me.token_domain_id' },
+				'retired' => 0,
 			},{
 				join => 'token_domain', alias => 'total_count'
 			})->count_rs->as_query,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ddg_community_platform",
-  "version": "0.210.0",
+  "version": "0.215.0",
   "engines": {
     "node": ">=0.10.0"
   },