feat(components): restructure components view to show only latest com…

…ponents for performance reasons

rename Display Filter to Quick Filter and Filters to Advanced Search
use new datatables search api throughout the application

backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java

@@ -159,11 +159,6 @@ public List<Release> getReleaseSummary() throws TException {
         return releases;
     }
 
-    public List<Component> getRecentComponents() {
-        return componentRepository.getRecentComponents();
-    }
-
-
     public List<Release> getRecentReleases() {
         return releaseRepository.getRecentReleases();
     }
@@ -992,4 +987,12 @@ public Map<String,List<String>> getDuplicateReleaseSources() {
 
         return CommonUtils.getIdentifierToListOfDuplicates(releaseIdentifierToReleaseId);
     }
+
+    public List<Component> getRecentComponentsSummary(int limit, User user) {
+        return componentRepository.getRecentComponentsSummary(limit, user);
+    }
+
+    public int getTotalComponentsCount() {
+        return componentRepository.getTotalComponentsCount();
+    }
 }

backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentRepository.java

@@ -18,6 +18,7 @@
 import org.eclipse.sw360.datahandler.thrift.components.Component;
 import org.eclipse.sw360.datahandler.thrift.users.User;
 import org.ektorp.ViewQuery;
+import org.ektorp.ViewResult;
 import org.ektorp.support.View;
 
 import java.util.HashSet;
@@ -99,21 +100,12 @@ public class ComponentRepository extends SummaryAwareRepository<Component> {
                     "        }" +
                     "    }";
 
-
     public ComponentRepository(DatabaseConnector db, ReleaseRepository releaseRepository, VendorRepository vendorRepository) {
         super(Component.class, db, new ComponentSummary(releaseRepository, vendorRepository));
 
         initStandardDesignDocument();
     }
 
-    @View(name = "recent", map = RECENT_VIEW)
-    public List<Component> getRecentComponents() {
-        ViewQuery query = createQuery("recent");
-        // Get the 5 last documents
-        query.limit(5).descending(true).includeDocs(false);
-        return makeSummary(SummaryType.SHORT, queryForIds(query));
-    }
-
     @View(name = "usedAttachmentContents", map = USED_ATTACHMENT_CONTENT_IDS)
     public Set<String> getUsedAttachmentContents() {
         return queryForIdsAsValue(createQuery("usedAttachmentContents"));
@@ -146,6 +138,17 @@ public List<Component> getComponentSummary(User user) {
         return makeSummaryWithPermissionsFromFullDocs(SummaryType.SUMMARY, componentList, user);
     }
 
+    @View(name = "recent", map = RECENT_VIEW)
+    public List<Component> getRecentComponentsSummary(int limit, User user) {
+        ViewQuery query = createQuery("recent").includeDocs(true).descending(true);
+        if (limit >= 0){
+            query.limit(limit);
+        }
+        List<Component> components = db.queryView(query, Component.class);
+
+        return makeSummaryWithPermissionsFromFullDocs(SummaryType.SUMMARY, components, user);
+    }
+
     @View(name = "byname", map = BY_NAME_VIEW)
     public Set<String> getMyComponentIdsByName(String name) {
         return queryForIdsAsValue("byname", name);
@@ -175,4 +178,10 @@ public Set<Component> getUsingComponents(Set<String> releaseIds) {
         final Set<String> componentIdsByLinkingRelease = queryForIdsAsValue("byLinkingRelease", releaseIds);
         return new HashSet<>(get(componentIdsByLinkingRelease));
     }
+
+    public int getTotalComponentsCount(){
+        ViewQuery query = createQuery("all").includeDocs(false).limit(0);
+        ViewResult result = db.queryView(query);
+        return result.getTotalRows();
+    }
 }

backend/src/src-components/src/main/java/org/eclipse/sw360/components/ComponentHandler.java

@@ -72,6 +72,19 @@ public List<Component> getComponentSummary(User user) throws TException {
         return handler.getComponentSummary(user);
     }
 
+    @Override
+    public List<Component> getRecentComponentsSummary(int limit, User user) throws TException {
+        assertUser(user);
+
+        return handler.getRecentComponentsSummary(limit, user);
+    }
+
+    @Override
+    public int getTotalComponentsCount(User user) throws TException {
+        assertUser(user);
+        return handler.getTotalComponentsCount();
+    }
+
     @Override
     public List<Release> getReleaseSummary(User user) throws TException {
         assertUser(user);
@@ -111,11 +124,6 @@ public List<Release> getSubscribedReleases(User user) throws TException {
         return handler.getSubscribedReleases(user.getEmail());
     }
 
-    @Override
-    public List<Component> getRecentComponents() throws TException {
-        return handler.getRecentComponents();
-    }
-
     @Override
     public List<Release> getRecentReleases() throws TException {
         return handler.getRecentReleases();

backend/src/src-components/src/test/java/org/eclipse/sw360/components/db/ComponentDatabaseHandlerTest.java

@@ -92,16 +92,16 @@ public void setUp() throws Exception {
 
 
         components = new ArrayList<>();
-        Component component1 = new Component().setId("C1").setName("component1").setDescription("d1").setCreatedBy(email1).setMainLicenseIds(new HashSet<>(Arrays.asList("lic1")));
+        Component component1 = new Component().setId("C1").setName("component1").setDescription("d1").setCreatedBy(email1).setMainLicenseIds(new HashSet<>(Arrays.asList("lic1"))).setCreatedOn("2017-07-20");
         component1.addToReleaseIds("R1A");
         component1.addToReleaseIds("R1B");
         components.add(component1);
-        Component component2 = new Component().setId("C2").setName("component2").setDescription("d2").setCreatedBy(email2).setMainLicenseIds(new HashSet<>(Arrays.asList("lic2")));
+        Component component2 = new Component().setId("C2").setName("component2").setDescription("d2").setCreatedBy(email2).setMainLicenseIds(new HashSet<>(Arrays.asList("lic2"))).setCreatedOn("2017-07-21");
         component2.addToReleaseIds("R2A");
         component2.addToReleaseIds("R2B");
         component2.addToReleaseIds("R2C");
         components.add(component2);
-        Component component3 = new Component().setId("C3").setName("component3").setDescription("d3").setCreatedBy(email1).setMainLicenseIds(new HashSet<>(Arrays.asList("lic3")));
+        Component component3 = new Component().setId("C3").setName("component3").setDescription("d3").setCreatedBy(email1).setMainLicenseIds(new HashSet<>(Arrays.asList("lic3"))).setCreatedOn("2017-07-22");
         component3.addToSubscribers(email1);
         component3.addToLanguages("E");
         components.add(component3);
@@ -246,9 +246,17 @@ public void testComponentSummary() throws Exception {
 
     @Test
     public void testGetRecentComponents() throws Exception {
-        List<Component> recentComponents = handler.getRecentComponents();
+        List<Component> recentComponents = handler.getRecentComponentsSummary(5, user1);
         Iterable<String> componentIds = getComponentIds(recentComponents);
-        assertThat(componentIds, containsInAnyOrder("C1", "C2", "C3"));
+        assertThat(componentIds, contains("C3", "C2", "C1"));
+    }
+
+    @Test
+    public void testGetRecentComponents2() throws Exception {
+        List<Component> recentComponents = handler.getRecentComponentsSummary(2, user1);
+        Iterable<String> componentIds = getComponentIds(recentComponents);
+        assertEquals(2, recentComponents.size());
+        assertThat(componentIds, contains("C3", "C2"));
     }
 
     @Test

docs/architecture/adr-001.md

@@ -0,0 +1,51 @@
+[//]: <> (Copyright Siemens AG, 2017.)
+[//]: <> (Part of the SW360 Portal Project.)
+[//]: <> ()
+[//]: <> (SPDX-License-Identifier: EPL-1.0)
+[//]: <> ()
+[//]: <> (All rights reserved. This program and the accompanying materials)
+[//]: <> (are made available under the terms of the Eclipse Public License v1.0)
+[//]: <> (which accompanies this distribution, and is available at)
+[//]: <> (http://www.eclipse.org/legal/epl-v10.html)
+
+## ADR 1: Pagination in Backend
+
+### Context
+  Components and Projects pages display entire contents of the database in one table. Reading data, generating HTML 
+  page, and rendering the table (which is paginated by datatables on the client) all take a long time with thousands of 
+  entities.
+
+  Datatables supports server-side processing for displaying and paginating data. For that, the server must be able to
+  tell datatables how many rows are in the table in total and to load a page of data starting from given index.
+
+  However, CouchDB cookbook strongly discourages loading of data starting from some index because of performance 
+  concerns. Instead, loading data starting from a specific key should be used. This is incompatible with what datatables
+  requires and also makes going to previous pages highly complicated.
+
+  To support sorting of the table by multiple columns would require creating a CouchDB view per column.
+
+  In addition, projects are filtered by visibility in backend after loading from CouchDB. This filtering cannot be
+  implemented in CouchDB. This complicates the matters even further with regards to pagination of projects table in
+  backend.
+
+### Decision
+  We will not use datatables' server-side processing as it's not worth the effort.
+
+  We will load only some number of latest components by default and let the user increase that number up to all
+  available components. We will make this choice sticky between sessions.
+
+  We will not change the projects table for now as the users have the option of loading only the projects from their
+  group and are not disturbed much by the performance of the page when all projects are displayed.
+
+### Status
+  Accepted
+
+### Consequences
+  Users will not see some components that are already created in the system and may try to create the "missing"
+  components. Quick Filter will not help them find the component as it works on client-side only.
+  To really make sure that a component is not in the system, users will have to use Advanced Search.
+
+  Loading time of the components page (with default settings) will improve dramatically and will be independent of
+  the total number of components in the system.
+
+  Loading time of unfiltered projects table will still be slow with thousands of projects.

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortalConstants.java

@@ -63,6 +63,8 @@ public class PortalConstants {
     public static final String SELECTED_TAB = "selectedTab";
     public static final String IS_USER_AT_LEAST_CLEARING_ADMIN = "isUserAtLeastClearingAdmin";
     public static final String DOCUMENT_TYPE = "documentType";
+    public static final String VIEW_SIZE = "viewSize";
+    public static final String TOTAL_ROWS = "totalRows";
 
     //! Specialized keys for licenses
     public static final String KEY_LICENSE_DETAIL = "licenseDetail";
@@ -182,6 +184,7 @@ public class PortalConstants {
 
     //! Specialized keys for users
     public static final String CUSTOM_FIELD_PROJECT_GROUP_FILTER = "ProjectGroupFilter";
+    public static final String CUSTOM_FIELD_COMPONENTS_VIEW_SIZE = "ComponentsViewSize";
 
     //! Specialized keys for scheduling
     public static final String CVESEARCH_IS_SCHEDULED = "cveSearchIsScheduled";

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortletUtils.java

@@ -11,9 +11,25 @@
 package org.eclipse.sw360.portal.common;
 
 import com.google.common.collect.Sets;
+import com.liferay.portal.kernel.exception.PortalException;
+import com.liferay.portal.kernel.exception.SystemException;
 import com.liferay.portal.kernel.json.JSONArray;
 import com.liferay.portal.kernel.json.JSONFactoryUtil;
 import com.liferay.portal.kernel.json.JSONObject;
+import com.liferay.portal.kernel.util.WebKeys;
+import com.liferay.portal.model.ResourceConstants;
+import com.liferay.portal.model.Role;
+import com.liferay.portal.model.RoleConstants;
+import com.liferay.portal.security.permission.ActionKeys;
+import com.liferay.portal.service.ResourcePermissionLocalServiceUtil;
+import com.liferay.portal.service.RoleLocalServiceUtil;
+import com.liferay.portal.service.UserLocalServiceUtil;
+import com.liferay.portal.theme.ThemeDisplay;
+import com.liferay.portlet.expando.model.ExpandoBridge;
+import com.liferay.portlet.expando.model.ExpandoColumn;
+import com.liferay.portlet.expando.model.ExpandoColumnConstants;
+import com.liferay.portlet.expando.model.ExpandoTableConstants;
+import com.liferay.portlet.expando.service.ExpandoColumnLocalServiceUtil;
 import org.eclipse.sw360.datahandler.common.CommonUtils;
 import org.eclipse.sw360.datahandler.common.SW360Utils;
 import org.eclipse.sw360.datahandler.thrift.*;
@@ -44,6 +60,8 @@
 import static org.eclipse.sw360.datahandler.common.CommonUtils.isNullEmptyOrWhitespace;
 import static org.eclipse.sw360.datahandler.common.CommonUtils.nullToEmptyList;
 import static java.lang.Integer.parseInt;
+import static org.eclipse.sw360.portal.common.PortalConstants.CUSTOM_FIELD_COMPONENTS_VIEW_SIZE;
+import static org.eclipse.sw360.portal.common.PortalConstants.CUSTOM_FIELD_PROJECT_GROUP_FILTER;
 
 /**
  * Portlet helpers
@@ -340,4 +358,40 @@ public static Map<String,Set<String>> getCustomMapFromRequest(PortletRequest req
         }
         return customMap;
     }
+
+    private static com.liferay.portal.model.User getLiferayUser(PortletRequest request, User user) throws PortalException, SystemException {
+        ThemeDisplay themeDisplay = (ThemeDisplay) request.getAttribute(WebKeys.THEME_DISPLAY);
+        long companyId = themeDisplay.getCompanyId();
+        return UserLocalServiceUtil.getUserByEmailAddress(companyId, user.email);
+    }
+
+    static void ensureUserCustomFieldExists(com.liferay.portal.model.User liferayUser, PortletRequest request, String customFieldName, int customFieldType) throws PortalException, SystemException {
+        ExpandoBridge exp = liferayUser.getExpandoBridge();
+        if (!exp.hasAttribute(customFieldName)){
+            exp.addAttribute(customFieldName, customFieldType, false);
+            long companyId = liferayUser.getCompanyId();
+
+            ExpandoColumn column = ExpandoColumnLocalServiceUtil.getColumn(companyId, exp.getClassName(), ExpandoTableConstants.DEFAULT_TABLE_NAME, customFieldName);
+
+            String[] roleNames = new String[]{RoleConstants.USER, RoleConstants.POWER_USER};
+            for (String roleName: roleNames) {
+                Role role = RoleLocalServiceUtil.getRole(companyId, roleName);
+                if (role != null && column != null) {
+                    ResourcePermissionLocalServiceUtil.setResourcePermissions(companyId,
+                            ExpandoColumn.class.getName(),
+                            ResourceConstants.SCOPE_INDIVIDUAL,
+                            String.valueOf(column.getColumnId()),
+                            role.getRoleId(),
+                            new String[]{ActionKeys.VIEW, ActionKeys.UPDATE});
+                }
+            }
+        }
+    }
+
+    public static ExpandoBridge getUserExpandoBridge(PortletRequest request, User user) throws PortalException, SystemException {
+        com.liferay.portal.model.User liferayUser = getLiferayUser(request, user);
+        ensureUserCustomFieldExists(liferayUser, request, CUSTOM_FIELD_PROJECT_GROUP_FILTER, ExpandoColumnConstants.STRING);
+        ensureUserCustomFieldExists(liferayUser, request, CUSTOM_FIELD_COMPONENTS_VIEW_SIZE, ExpandoColumnConstants.INTEGER);
+        return liferayUser.getExpandoBridge();
+    }
 }

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java

@@ -16,6 +16,8 @@
 import com.liferay.portal.kernel.json.JSONObject;
 import com.liferay.portal.kernel.portlet.PortletResponseUtil;
 import com.liferay.portal.kernel.servlet.SessionMessages;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
 import org.eclipse.sw360.datahandler.common.CommonUtils;
 import org.eclipse.sw360.datahandler.common.SW360Constants;
 import org.eclipse.sw360.datahandler.common.SW360Utils;
@@ -42,8 +44,6 @@
 import org.eclipse.sw360.portal.portlets.FossologyAwarePortlet;
 import org.eclipse.sw360.portal.users.LifeRayUserSession;
 import org.eclipse.sw360.portal.users.UserCacheHolder;
-import org.apache.log4j.Logger;
-import org.apache.thrift.TException;
 
 import javax.portlet.*;
 import javax.servlet.http.HttpServletResponse;
@@ -54,9 +54,7 @@
 
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.base.Strings.nullToEmpty;
-import static org.eclipse.sw360.datahandler.common.CommonUtils.nullToEmptyList;
-import static org.eclipse.sw360.datahandler.common.CommonUtils.nullToEmptyMap;
-import static org.eclipse.sw360.datahandler.common.CommonUtils.nullToEmptySet;
+import static org.eclipse.sw360.datahandler.common.CommonUtils.*;
 import static org.eclipse.sw360.datahandler.common.SW360Constants.CONTENT_TYPE_OPENXML_SPREADSHEET;
 import static org.eclipse.sw360.datahandler.common.SW360Utils.printName;
 import static org.eclipse.sw360.portal.common.PortalConstants.*;
@@ -749,10 +747,12 @@ private List<Component> getFilteredComponentList(PortletRequest request) throws
 
         try {
             final User user = UserCacheHolder.getUserFromRequest(request);
+            int limit = loadAndStoreStickyViewSize(request, user);
             ComponentService.Iface componentClient = thriftClients.makeComponentClient();
+            request.setAttribute(PortalConstants.TOTAL_ROWS, componentClient.getTotalComponentsCount(user));
 
             if (filterMap.isEmpty()) {
-                componentList = componentClient.getComponentSummary(user);
+                componentList = componentClient.getRecentComponentsSummary(limit, user);
             } else {
                 componentList = componentClient.refineSearch(null, filterMap);
             }
@@ -763,6 +763,19 @@ private List<Component> getFilteredComponentList(PortletRequest request) throws
         return componentList;
     }
 
+    private int loadAndStoreStickyViewSize(PortletRequest request, User user) {
+        String view_size = request.getParameter(PortalConstants.VIEW_SIZE);
+        final int limit;
+        if (isNullEmptyOrWhitespace(view_size)){
+            limit = ComponentPortletUtils.loadStickyViewSize(request, user);
+        } else {
+            limit = Integer.parseInt(view_size);
+            ComponentPortletUtils.saveStickyViewSize(request, user, limit);
+        }
+        request.setAttribute(PortalConstants.VIEW_SIZE, limit);
+        return limit;
+    }
+
     //! Actions
     @UsedAsLiferayAction
     public void updateComponent(ActionRequest request, ActionResponse response) throws PortletException, IOException {