Merge branch 'usage-ping-port' into 'master'

Usage ping port

Closes #27750

See merge request !10481

app/assets/javascripts/dispatcher.js

@@ -367,6 +367,9 @@ const ShortcutsBlob = require('./shortcuts_blob');
         case 'admin':
           new Admin();
           switch (path[1]) {
+            case 'cohorts':
+              new gl.UsagePing();
+              break;
             case 'groups':
               new UsersSelect();
               break;

app/assets/javascripts/main.js

@@ -165,6 +165,7 @@ import './syntax_highlight';
 import './task_list';
 import './todos';
 import './tree';
+import './usage_ping';
 import './user';
 import './user_tabs';
 import './username_validator';

app/assets/javascripts/usage_ping.js

@@ -0,0 +1,15 @@
+function UsagePing() {
+  const usageDataUrl = $('.usage-data').data('endpoint');
+
+  $.ajax({
+    type: 'GET',
+    url: usageDataUrl,
+    dataType: 'html',
+    success(html) {
+      $('.usage-data').html(html);
+    },
+  });
+}
+
+window.gl = window.gl || {};
+window.gl.UsagePing = UsagePing;

app/controllers/admin/application_settings_controller.rb

@@ -17,6 +17,18 @@ def update
     end
   end
 
+  def usage_data
+    respond_to do |format|
+      format.html do
+        usage_data = Gitlab::UsageData.data
+        usage_data_json = params[:pretty] ? JSON.pretty_generate(usage_data) : usage_data.to_json
+
+        render html: Gitlab::Highlight.highlight('payload.json', usage_data_json)
+      end
+      format.json { render json: Gitlab::UsageData.to_json }
+    end
+  end
+
   def reset_runners_token
     @application_setting.reset_runners_registration_token!
     flash[:notice] = 'New runners registration token has been generated!'
@@ -135,6 +147,7 @@ def application_setting_params_ce
       :version_check_enabled,
       :terminal_max_session_time,
       :polling_interval_multiplier,
+      :usage_ping_enabled,
 
       disabled_oauth_sign_in_sources: [],
       import_sources: [],

app/controllers/admin/cohorts_controller.rb

@@ -0,0 +1,11 @@
+class Admin::CohortsController < Admin::ApplicationController
+  def index
+    if current_application_settings.usage_ping_enabled
+      cohorts_results = Rails.cache.fetch('cohorts', expires_in: 1.day) do
+        CohortsService.new.execute
+      end
+
+      @cohorts = CohortsSerializer.new.represent(cohorts_results)
+    end
+  end
+end

app/controllers/projects/git_http_controller.rb

@@ -5,6 +5,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
   # GET /foo/bar.git/info/refs?service=git-receive-pack (git push)
   def info_refs
     if upload_pack? && upload_pack_allowed?
+      log_user_activity
+
       render_ok
     elsif receive_pack? && receive_pack_allowed?
       render_ok
@@ -106,4 +108,8 @@ def receive_pack_allowed?
   def access_klass
     @access_klass ||= wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess
   end
+
+  def log_user_activity
+    Users::ActivityService.new(user, 'pull').execute
+  end
 end

app/controllers/sessions_controller.rb

@@ -35,6 +35,7 @@ def create
       # hide the signed-in notification
       flash[:notice] = nil
       log_audit_event(current_user, with: authentication_method)
+      log_user_activity(current_user)
     end
   end
 
@@ -123,6 +124,10 @@ def log_audit_event(user, options = {})
       for_authentication.security_event
   end
 
+  def log_user_activity(user)
+    Users::ActivityService.new(user, 'login').execute
+  end
+
   def load_recaptcha
     Gitlab::Recaptcha.load_configurations!
   end

app/models/application_setting.rb

@@ -238,7 +238,8 @@ def self.defaults_ce
       terminal_max_session_time: 0,
       two_factor_grace_period: 48,
       user_default_external: false,
-      polling_interval_multiplier: 1
+      polling_interval_multiplier: 1,
+      usage_ping_enabled: true
     }
   end
 

app/serializers/cohort_activity_month_entity.rb

@@ -0,0 +1,11 @@
+class CohortActivityMonthEntity < Grape::Entity
+  include ActionView::Helpers::NumberHelper
+
+  expose :total do |cohort_activity_month|
+    number_with_delimiter(cohort_activity_month[:total])
+  end
+
+  expose :percentage do |cohort_activity_month|
+    number_to_percentage(cohort_activity_month[:percentage], precision: 0)
+  end
+end

app/serializers/cohort_entity.rb

@@ -0,0 +1,17 @@
+class CohortEntity < Grape::Entity
+  include ActionView::Helpers::NumberHelper
+
+  expose :registration_month do |cohort|
+    cohort[:registration_month].strftime('%b %Y')
+  end
+
+  expose :total do |cohort|
+    number_with_delimiter(cohort[:total])
+  end
+
+  expose :inactive do |cohort|
+    number_with_delimiter(cohort[:inactive])
+  end
+
+  expose :activity_months, using: CohortActivityMonthEntity
+end

app/serializers/cohorts_entity.rb

@@ -0,0 +1,4 @@
+class CohortsEntity < Grape::Entity
+  expose :months_included
+  expose :cohorts, using: CohortEntity
+end

app/serializers/cohorts_serializer.rb

@@ -0,0 +1,3 @@
+class CohortsSerializer < AnalyticsGenericSerializer
+  entity CohortsEntity
+end

app/services/cohorts_service.rb

@@ -0,0 +1,100 @@
+class CohortsService
+  MONTHS_INCLUDED = 12
+
+  def execute
+    {
+      months_included: MONTHS_INCLUDED,
+      cohorts: cohorts
+    }
+  end
+
+  # Get an array of hashes that looks like:
+  #
+  #     [
+  #       {
+  #         registration_month: Date.new(2017, 3),
+  #         activity_months: [3, 2, 1],
+  #         total: 3
+  #         inactive: 0
+  #      },
+  #      etc.
+  #
+  # The `months` array is always from oldest to newest, so it's always
+  # non-strictly decreasing from left to right.
+  def cohorts
+    months = Array.new(MONTHS_INCLUDED) { |i| i.months.ago.beginning_of_month.to_date }
+
+    Array.new(MONTHS_INCLUDED) do
+      registration_month = months.last
+      activity_months = running_totals(months, registration_month)
+
+      # Even if no users registered in this month, we always want to have a
+      # value to fill in the table.
+      inactive = counts_by_month[[registration_month, nil]].to_i
+
+      months.pop
+
+      {
+        registration_month: registration_month,
+        activity_months: activity_months,
+        total: activity_months.first[:total],
+        inactive: inactive
+      }
+    end
+  end
+
+  private
+
+  # Calculate a running sum of active users, so users active in later months
+  # count as active in this month, too. Start with the most recent month first,
+  # for calculating the running totals, and then reverse for displaying in the
+  # table.
+  #
+  # Each month has a total, and a percentage of the overall total, as keys.
+  def running_totals(all_months, registration_month)
+    month_totals =
+      all_months
+        .map { |activity_month| counts_by_month[[registration_month, activity_month]] }
+        .reduce([]) { |result, total| result << result.last.to_i + total.to_i }
+        .reverse
+
+    overall_total = month_totals.first
+
+    month_totals.map do |total|
+      { total: total, percentage: total.zero? ? 0 : 100 * total / overall_total }
+    end
+  end
+
+  # Get a hash that looks like:
+  #
+  #     {
+  #       [created_at_month, last_activity_on_month] => count,
+  #       [created_at_month, last_activity_on_month_2] => count_2,
+  #       # etc.
+  #     }
+  #
+  # created_at_month can never be nil, but last_activity_on_month can (when a
+  # user has never logged in, just been created). This covers the last
+  # MONTHS_INCLUDED months.
+  def counts_by_month
+    @counts_by_month ||=
+      begin
+        created_at_month = column_to_date('created_at')
+        last_activity_on_month = column_to_date('last_activity_on')
+
+        User
+          .where('created_at > ?', MONTHS_INCLUDED.months.ago.end_of_month)
+          .group(created_at_month, last_activity_on_month)
+          .reorder("#{created_at_month} ASC", "#{last_activity_on_month} ASC")
+          .count
+      end
+  end
+
+  def column_to_date(column)
+    if Gitlab::Database.postgresql?
+      "CAST(DATE_TRUNC('month', #{column}) AS date)"
+    else
+      "STR_TO_DATE(DATE_FORMAT(#{column}, '%Y-%m-01'), '%Y-%m-%d')"
+    end
+  end
+end

app/services/event_create_service.rb

@@ -72,6 +72,8 @@ def create_project(project, current_user)
 
   def push(project, current_user, push_data)
     create_event(project, current_user, Event::PUSHED, data: push_data)
+
+    Users::ActivityService.new(current_user, 'push').execute
   end
 
   private

app/services/users/activity_service.rb

@@ -0,0 +1,22 @@
+module Users
+  class ActivityService
+    def initialize(author, activity)
+      @author = author.respond_to?(:user) ? author.user : author
+      @activity = activity
+    end
+
+    def execute
+      return unless @author && @author.is_a?(User)
+
+      record_activity
+    end
+
+    private
+
+    def record_activity
+      Gitlab::UserActivities.record(@author.id)
+
+      Rails.logger.debug("Recorded activity: #{@activity} for User ID: #{@author.id} (username: #{@author.username}")
+    end
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -477,7 +477,7 @@
           diagrams in Asciidoc documents using an external PlantUML service.
 
   %fieldset
-    %legend Usage statistics
+    %legend#usage-statistics Usage statistics
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox
@@ -486,6 +486,19 @@
             Version check enabled
           .help-block
             Let GitLab inform you when an update is available.
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :usage_ping_enabled do
+            = f.check_box :usage_ping_enabled
+            Usage ping enabled
+            = link_to icon('question-circle'), help_page_path("user/admin_area/settings/usage_statistics", anchor: "usage-data")
+          .help-block
+            Every week GitLab will report license usage back to GitLab, Inc.
+            Disable this option if you do not want this to occur. To see the
+            JSON payload that will be sent, visit the
+            = succeed '.' do
+              = link_to "Cohorts page", admin_cohorts_path(anchor: 'usage-ping')
 
   %fieldset
     %legend Email

app/views/admin/cohorts/_cohorts_table.html.haml

@@ -0,0 +1,28 @@
+.bs-callout.clearfix
+  %p
+    User cohorts are shown for the last #{@cohorts[:months_included]}
+    months. Only users with activity are counted in the cohort total; inactive
+    users are counted separately.
+    = link_to icon('question-circle'), help_page_path('user/admin_area/user_cohorts', anchor: 'cohorts'), title: 'About this feature', target: '_blank'
+
+.table-holder
+  %table.table
+    %thead
+      %tr
+        %th Registration month
+        %th Inactive users
+        %th Cohort total
+        - @cohorts[:months_included].times do |i|
+          %th Month #{i}
+    %tbody
+      - @cohorts[:cohorts].each do |cohort|
+        %tr
+          %td= cohort[:registration_month]
+          %td= cohort[:inactive]
+          %td= cohort[:total]
+          - cohort[:activity_months].each do |activity_month|
+            %td
+              - next if cohort[:total] == '0'
+              = activity_month[:percentage]
+              %br
+              = activity_month[:total]

app/views/admin/cohorts/_usage_ping.html.haml

@@ -0,0 +1,10 @@
+%h2#usage-ping Usage ping
+
+.bs-callout.clearfix
+  %p
+    User cohorts are shown because the usage ping is enabled. The data sent with
+    this is shown below. To disable this, visit
+    = succeed '.' do
+      = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics')
+
+%pre.usage-data.js-syntax-highlight.code.highlight{ data: { endpoint: usage_data_admin_application_settings_path(format: :html, pretty: true) } }

app/views/admin/cohorts/index.html.haml

@@ -0,0 +1,16 @@
+- @no_container = true
+= render "admin/dashboard/head"
+
+%div{ class: container_class }
+  - if @cohorts
+    = render 'cohorts_table'
+    = render 'usage_ping'
+  - else
+    .bs-callout.bs-callout-warning.clearfix
+      %p
+        User cohorts are only shown when the
+        = link_to 'usage ping', help_page_path('user/admin_area/usage_statistics'), target: '_blank'
+        is enabled. To enable it and see user cohorts,
+        visit
+        = succeed '.' do
+          = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics')

app/views/admin/dashboard/_head.html.haml

@@ -27,3 +27,7 @@
           = link_to admin_runners_path, title: 'Runners' do
             %span
               Runners
+        = nav_link path: 'cohorts#index' do
+          = link_to admin_cohorts_path, title: 'Cohorts' do
+            %span
+              Cohorts