Skip to content
This repository has been archived by the owner on Oct 11, 2019. It is now read-only.

Commit

Permalink
only show tags in tags dropdown list that have been used on pages tha…
Browse files Browse the repository at this point in the history
…t are actually visible to the user; speed up PagePermissionFilter
  • Loading branch information
blizzy78 committed Aug 17, 2012
1 parent 914a977 commit dcaa643
Show file tree
Hide file tree
Showing 9 changed files with 297 additions and 93 deletions.
Expand Up @@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -30,6 +31,8 @@
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;

import com.google.common.collect.Sets;

import de.blizzy.documentr.access.GrantedAuthorityTarget.Type;
import de.blizzy.documentr.page.IPageStore;
import de.blizzy.documentr.page.PageNotFoundException;
Expand Down Expand Up @@ -211,6 +214,22 @@ public boolean hasPagePermissionInOtherBranches(Authentication authentication, S
return false;
}

public Set<String> getBranchesForPermission(Authentication authentication, Permission permission) {
try {
Set<String> branches = Sets.newHashSet();
for (String project : repoManager.listProjects()) {
for (String branch : repoManager.listProjectBranches(project)) {
if (hasBranchPermission(authentication, project, branch, permission)) {
branches.add(project + "/" + branch); //$NON-NLS-1$
}
}
}
return branches;
} catch (IOException e) {
throw new AuthenticationServiceException(e.getMessage(), e);
}
}

private boolean hasRoleOnBranch(Authentication authentication, String projectName, String branchName,
String roleName) throws IOException {

Expand Down
@@ -0,0 +1,30 @@
/*
documentr - Edit, maintain, and present software documentation on the web.
Copyright (C) 2012 Maik Schreiber
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package de.blizzy.documentr.search;

import java.util.BitSet;

import org.apache.lucene.search.Collector;

abstract class AbstractDocIdsCollector extends Collector {
protected BitSet docIds = new BitSet();

BitSet getDocIds() {
return docIds;
}
}
@@ -0,0 +1,44 @@
/*
documentr - Edit, maintain, and present software documentation on the web.
Copyright (C) 2012 Maik Schreiber
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package de.blizzy.documentr.search;

import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;

class AllDocIdsCollector extends AbstractDocIdsCollector {
private int docBase;

@Override
public void setScorer(Scorer scorer) {
}

@Override
public void collect(int doc) {
docIds.set(docBase + doc);
}

@Override
public void setNextReader(AtomicReaderContext context) {
docBase = context.docBase;
}

@Override
public boolean acceptsDocsOutOfOrder() {
return true;
}
}
@@ -0,0 +1,75 @@
/*
documentr - Edit, maintain, and present software documentation on the web.
Copyright (C) 2012 Maik Schreiber
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package de.blizzy.documentr.search;

import java.io.IOException;
import java.util.Set;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.springframework.security.core.Authentication;

import com.google.common.collect.Sets;

import de.blizzy.documentr.access.DocumentrPermissionEvaluator;
import de.blizzy.documentr.access.Permission;

class InaccessibleDocIdsCollector extends AbstractDocIdsCollector {
private static final Set<String> FIELDS = Sets.newHashSet(PageIndex.PROJECT, PageIndex.BRANCH, PageIndex.PATH);

private Authentication authentication;
private Permission permission;
private DocumentrPermissionEvaluator permissionEvaluator;
private AtomicReader reader;
private int docBase;

InaccessibleDocIdsCollector(Authentication authentication, Permission permission,
DocumentrPermissionEvaluator permissionEvaluator) {

this.authentication = authentication;
this.permission = permission;
this.permissionEvaluator = permissionEvaluator;
}

@Override
public void setScorer(org.apache.lucene.search.Scorer scorer) {
}

@Override
public void setNextReader(AtomicReaderContext context) {
reader = context.reader();
docBase = context.docBase;
}

@Override
public void collect(int doc) throws IOException {
Document document = reader.document(doc, FIELDS);
String projectName = document.get(PageIndex.PROJECT);
String branchName = document.get(PageIndex.BRANCH);
String path = document.get(PageIndex.PATH);
if (!permissionEvaluator.hasPagePermission(authentication, projectName, branchName, path, permission)) {
docIds.set(docBase + doc);
}
}

@Override
public boolean acceptsDocsOutOfOrder() {
return true;
}
}
Expand Up @@ -20,6 +20,7 @@
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -52,6 +53,7 @@
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
Expand All @@ -63,10 +65,14 @@
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Formatter;
import org.apache.lucene.search.highlight.Highlighter;
Expand All @@ -82,7 +88,9 @@
import org.apache.lucene.search.spell.SuggestWord;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.Version;
import org.cyberneko.html.HTMLEntities;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -125,15 +133,17 @@ private static final class WordPosition {
}
}

static final String PROJECT = "project"; //$NON-NLS-1$
static final String BRANCH = "branch"; //$NON-NLS-1$
static final String PATH = "path"; //$NON-NLS-1$

private static final String FULL_PATH = "fullPath"; //$NON-NLS-1$
private static final String PROJECT = "project"; //$NON-NLS-1$
private static final String BRANCH = "branch"; //$NON-NLS-1$
private static final String PATH = "path"; //$NON-NLS-1$
private static final String TAG = "tag"; //$NON-NLS-1$
private static final String TITLE = "title"; //$NON-NLS-1$
private static final String TEXT = "text"; //$NON-NLS-1$
private static final String ALL_TEXT = "allText"; //$NON-NLS-1$
private static final String ALL_TEXT_SUGGESTIONS = "allTextSuggestions"; //$NON-NLS-1$
private static final String VIEW_RESTRICTION_ROLE = "viewRestrictionRole"; //$NON-NLS-1$
private static final int HITS_PER_PAGE = 20;
private static final int NUM_FRAGMENTS = 5;
private static final int FRAGMENT_SIZE = 50;
Expand Down Expand Up @@ -167,6 +177,8 @@ private static final class WordPosition {
private IPageStore pageStore;
@Autowired
private GlobalRepositoryManager repoManager;
@Autowired
private UserStore userStore;
private Analyzer defaultAnalyzer;
private Analyzer analyzer;
private File pageIndexDir;
Expand Down Expand Up @@ -316,6 +328,10 @@ private void addPageAsync(String projectName, String branchName, String path) th
for (String tag : page.getTags()) {
doc.add(new StringField(TAG, tag, Store.YES));
}
String viewRestrictionRole = page.getViewRestrictionRole();
if (StringUtils.isNotBlank(viewRestrictionRole)) {
doc.add(new StringField(VIEW_RESTRICTION_ROLE, viewRestrictionRole, Store.NO));
}
doc.add(new TextField(TITLE, page.getTitle(), Store.YES));
doc.add(new TextField(TEXT, text, Store.YES));
doc.add(new TextField(ALL_TEXT, page.getTitle(), Store.NO));
Expand Down Expand Up @@ -432,7 +448,8 @@ private SearchResult findPages(String searchText, int page, Authentication authe
Query query = parser.parse(searchText);
List<SearchHit> hits = Lists.newArrayList();
IndexReader reader = searcher.getIndexReader();
Filter filter = new PagePermissionFilter(authentication, Permission.VIEW, permissionEvaluator);
Bits visibleDocIds = getVisibleDocIds(searcher, authentication);
Filter filter = new PagePermissionFilter(visibleDocIds);
TopDocs docs = searcher.search(query, filter, HITS_PER_PAGE * page);
Formatter formatter = new SimpleHTMLFormatter("<strong>", "</strong>"); //$NON-NLS-1$ //$NON-NLS-2$
Scorer scorer = new QueryScorer(query);
Expand Down Expand Up @@ -540,32 +557,71 @@ private void cleanupFragments(String[] fragments) {
}
}

public Set<String> getAllTags() throws IOException {
DirectoryReader reader = null;
public Set<String> getAllTags(Authentication authentication) throws IOException {
IndexReader reader = null;
IndexSearcher searcher = null;
try {
if (alwaysRefresh) {
refreshBlocking();
}

// TODO: does not honor permissions - returns all tags regardless of whether they are only
// used in documents the user can't see or not

reader = readerManager.acquire();
Terms terms = MultiFields.getTerms(reader, TAG);
searcher = searcherManager.acquire();
Bits visibleDocs = getVisibleDocIds(searcher, authentication);
Set<String> tags = Sets.newHashSet();
if (terms != null) {
TermsEnum termsEnum = terms.iterator(null);
BytesRef ref;
while ((ref = termsEnum.next()) != null) {
tags.add(ref.utf8ToString());
if (visibleDocs.length() > 0) {
reader = searcher.getIndexReader();
Terms terms = MultiFields.getTerms(reader, TAG);
if (terms != null) {
TermsEnum termsEnum = terms.iterator(null);
BytesRef ref;
while ((ref = termsEnum.next()) != null) {
DocsEnum docsEnum = termsEnum.docs(visibleDocs, null, 0);
if (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
tags.add(ref.utf8ToString());
}
}
}
}
return tags;
} finally {
if (reader != null) {
readerManager.release(reader);
if (searcher != null) {
searcherManager.release(searcher);
}
}
}

private Bits getVisibleDocIds(IndexSearcher searcher, Authentication authentication) throws IOException {
Set<String> branches = permissionEvaluator.getBranchesForPermission(authentication, Permission.VIEW);
BitSet docIds = new BitSet();
if (!branches.isEmpty()) {
// collect document IDs of all visible documents (via view permission on branches)
BooleanQuery allBranchesQuery = new BooleanQuery();
for (String projectAndBranch : branches) {
String projectName = StringUtils.substringBefore(projectAndBranch, "/"); //$NON-NLS-1$
String branchName = StringUtils.substringAfter(projectAndBranch, "/"); //$NON-NLS-1$
TermQuery projectQuery = new TermQuery(new Term(PROJECT, projectName));
TermQuery branchQuery = new TermQuery(new Term(BRANCH, branchName));
BooleanQuery projectAndBranchQuery = new BooleanQuery();
projectAndBranchQuery.add(projectQuery, BooleanClause.Occur.MUST);
projectAndBranchQuery.add(branchQuery, BooleanClause.Occur.MUST);
allBranchesQuery.add(projectAndBranchQuery, BooleanClause.Occur.SHOULD);
}
AbstractDocIdsCollector collector = new AllDocIdsCollector();
searcher.search(allBranchesQuery, collector);
docIds = collector.getDocIds();

// remove all inaccessible document IDs (via view restriction roles on pages)
Set<String> roles = Sets.newHashSet(userStore.listRoles());
BooleanQuery allRolesQuery = new BooleanQuery();
for (String role : roles) {
TermQuery roleQuery = new TermQuery(new Term(VIEW_RESTRICTION_ROLE, role));
allRolesQuery.add(roleQuery, BooleanClause.Occur.SHOULD);
}
collector = new InaccessibleDocIdsCollector(authentication, Permission.VIEW, permissionEvaluator);
searcher.search(allRolesQuery, collector);
docIds.andNot(collector.getDocIds());
}
return new DocIdBitSet(docIds);
}

private void refresh() {
Expand Down

0 comments on commit dcaa643

Please sign in to comment.