export/import group data via json file

app/controllers/api/groups_controller.rb

@@ -37,6 +37,11 @@ def upload_photo
     respond_with_resource
   end
 
+  def export
+    service.export(group: load_and_authorize(:group), actor: current_user)
+    render json: { success: :ok }
+  end
+
   private
   def track_visit
     VisitService.record(group: resource, visit: current_visit, user: current_user)

app/jobs/group_export_job.rb

@@ -0,0 +1,11 @@
+class GroupExportJob < ActiveJob::Base
+  queue_as :low_priority
+
+  def perform(group, actor)
+    groups = actor.groups.where(id: group.all_groups)
+    filename = GroupExportService.export_filename_for(group)
+    GroupExportService.export(groups, filename)
+    document = Document.create(author: actor, file: File.open(filename, 'r'), title: filename)
+    UserMailer.group_export_ready(actor, group, document).deliver
+  end
+end

app/mailers/user_mailer.rb

@@ -61,6 +61,15 @@ def user_added_to_group(recipient, event)
                      locale: [@user.locale, @inviter.locale]
   end
 
+  def group_export_ready(recipient, group, document)
+    @user     = recipient
+    @document = document
+    send_single_mail to: @user.email,
+                     subject_key: "user_mailer.group_export.subject",
+                     subject_params: {group_name: group.name},
+                     locale: @user.locale
+  end
+
   def login(user:, token:)
     @user = user
     @token = token

app/models/ability/group.rb

@@ -2,7 +2,7 @@ module Ability::Group
   def initialize(user)
     super(user)
 
-    can :show, ::Group do |group|
+    can [:show], ::Group do |group|
       if user.is_admin?
         true
       elsif group.archived_at || group.is_guest_group?

app/models/comment.rb

@@ -5,6 +5,7 @@ class Comment < ApplicationRecord
   include HasMentions
   include HasDrafts
   include HasCreatedEvent
+  include HasEvents
 
   has_paper_trail only: [:body]
 

app/models/concerns/group_export_relations.rb

@@ -0,0 +1,115 @@
+module GroupExportRelations
+  extend ActiveSupport::Concern
+
+  included do
+    # polls
+    has_many :discussion_polls, through: :discussions
+    has_many :poll_options, through: :polls
+    has_many :poll_unsubscriptions, through: :polls
+    has_many :poll_did_not_votes, through: :polls
+    has_many :outcomes, through: :polls
+    has_many :stances, through: :polls
+    has_many :stance_choices, through: :stances
+
+    # documents
+    has_many :discussion_documents,        through: :discussions,        source: :documents
+    has_many :poll_documents,              through: :polls,              source: :documents
+    has_many :comment_documents,           through: :comments,           source: :documents
+    has_many :public_discussion_documents, through: :public_discussions, source: :documents
+    has_many :public_poll_documents,       through: :public_polls,       source: :documents
+    has_many :public_comment_documents,    through: :public_comments,    source: :documents
+
+    # reactions
+    has_many :discussion_reactions,        -> { joins(:user) }, through: :discussions, source: :reactions
+    has_many :poll_reactions,              -> { joins(:user) }, through: :polls,       source: :reactions
+    has_many :stance_reactions,            -> { joins(:user) }, through: :stances,     source: :reactions
+    has_many :comment_reactions,           -> { joins(:user) }, through: :comments,    source: :reactions
+    has_many :outcome_reactions,           -> { joins(:user) }, through: :outcomes,    source: :reactions
+
+    # readers
+    has_many :discussion_readers,  through: :discussions
+
+    # guest groups
+    has_many :discussion_guest_groups,     through: :discussions,        source: :guest_group
+    has_many :poll_guest_groups,           through: :polls,              source: :guest_group
+
+    # users
+    has_many :discussion_authors,      through: :discussions, source: :author
+    # has_many :discussion_reader_users, through: :discussion_readers, source: :user
+    has_many :comment_authors,         through: :comments,           source: :user
+    has_many :poll_authors,            through: :polls,              source: :author
+    has_many :outcome_authors,         through: :outcomes,           source: :author
+    has_many :stance_authors,          through: :stances,            source: :participant
+    has_many :reader_users,            through: :discussion_readers, source: :user
+    has_many :non_voters,              through: :poll_did_not_votes, source: :user
+
+    # events
+    has_many :membership_events,       through: :memberships, source: :events
+    has_many :discussion_events,       through: :discussions, source: :events
+    has_many :comment_events,          through: :comments,    source: :events
+    has_many :poll_events,             through: :polls,       source: :events
+    has_many :outcome_events,          through: :outcomes,    source: :events
+    has_many :stance_events,           through: :stances,     source: :events
+  end
+
+  def all_groups
+    Queries::UnionQuery.for(:groups, [
+      Group.where(id: self.id),
+      self.subgroups,
+      self.discussion_guest_groups,
+      self.poll_guest_groups
+    ])
+  end
+
+  def all_users
+    Queries::UnionQuery.for(:users, [
+      self.members,
+      self.discussion_authors,
+      self.comment_authors,
+      self.poll_authors,
+      self.outcome_authors,
+      self.stance_authors,
+      self.reaction_users,
+      self.reader_users,
+      self.non_voters
+    ])
+  end
+
+  def all_events
+    Queries::UnionQuery.for(:events, [
+      self.membership_events,
+      self.discussion_events,
+      self.comment_events,
+      self.poll_events,
+      self.outcome_events,
+      self.stance_events
+    ])
+  end
+
+  def all_notifications
+    Notification.where(event_id: all_events.pluck(:id))
+  end
+
+  def all_documents
+    Queries::UnionQuery.for(:documents, [
+      self.documents,
+      self.discussion_documents,
+      self.poll_documents,
+      self.comment_documents
+    ])
+  end
+
+  def all_reactions
+    Queries::UnionQuery.for(:reactions, [
+      self.discussion_reactions,
+      self.poll_reactions,
+      self.stance_reactions,
+      self.comment_reactions,
+      self.outcome_reactions
+    ])
+  end
+
+  def reaction_users
+    User.where(id: all_reactions.pluck(:user_id))
+  end
+end

app/models/discussion.rb

@@ -51,6 +51,7 @@ def self.always_versioned_fields
   has_many :polls, dependent: :destroy
   has_many :active_polls, -> { where(closed_at: nil) }, class_name: "Poll"
   has_one :search_vector
+
   has_many :comments, dependent: :destroy
   has_many :commenters, -> { uniq }, through: :comments, source: :user
   has_many :documents, as: :model, dependent: :destroy

app/models/discussion_reader.rb

@@ -61,9 +61,9 @@ def recall!(persist: true)
     save if persist
   end
 
-  def volume
+  def computed_volume
     if persisted?
-      super || membership&.volume || 'normal'
+      volume || membership&.volume || 'normal'
     else
       membership.volume
     end

app/models/formal_group.rb

@@ -48,7 +48,7 @@ class FormalGroup < Group
   belongs_to :default_group_cover
 
   has_many :subgroups,
-           -> { where(archived_at: nil).order(:name) },
+           -> { where(archived_at: nil) },
            class_name: 'Group',
            foreign_key: 'parent_id'
   has_many :all_subgroups, class_name: 'Group', foreign_key: :parent_id

app/models/group.rb

@@ -11,22 +11,29 @@ class Group < ApplicationRecord
 
   belongs_to :creator, class_name: 'User'
   belongs_to :parent, class_name: 'Group'
+
+  has_many :discussions,             foreign_key: :group_id, dependent: :destroy
+  has_many :public_discussions, -> { visible_to_public }, foreign_key: :group_id, dependent: :destroy, class_name: 'Discussion'
+  has_many :comments, through: :discussions
+
   has_many :all_memberships, dependent: :destroy, class_name: 'Membership'
   has_many :memberships, -> { where is_suspended: false, archived_at: nil }
+  has_many :members, through: :memberships, source: :user
+
   has_many :accepted_memberships, -> { accepted }, class_name: 'Membership'
+  has_many :accepted_members, through: :accepted_memberships, source: :user
+
   has_many :admin_memberships, -> { where admin: true, archived_at: nil }, class_name: 'Membership'
   has_many :admins, through: :admin_memberships, source: :user
 
   has_many :membership_requests, dependent: :destroy
   has_many :pending_membership_requests, -> { where response: nil }, class_name: 'MembershipRequest'
-  has_many :members, through: :memberships, source: :user
-  has_many :accepted_members, through: :accepted_memberships, source: :user
 
-  has_many :discussions, foreign_key: :group_id, dependent: :destroy
-  has_many :public_discussions, -> { visible_to_public }, foreign_key: :group_id, dependent: :destroy, class_name: 'Discussion'
   has_many :polls, foreign_key: :group_id, dependent: :destroy
   has_many :public_polls, through: :public_discussions, dependent: :destroy, source: :polls
 
+  include GroupExportRelations
+
   scope :archived, -> { where('archived_at IS NOT NULL') }
   scope :published, -> { where(archived_at: nil) }
 

app/models/guest_group.rb

@@ -5,6 +5,14 @@ def id_and_subgroup_ids
     Array(id)
   end
 
+  def subgroups
+    Group.none
+  end
+
+  def documents
+    Document.none
+  end
+
   def group_privacy=(term)
     raise 'guest groups cant be open' if term == 'open'
     super

app/models/membership.rb

@@ -89,6 +89,6 @@ def discussion_readers
   private
 
   def set_volume
-    self.volume = user.default_membership_volume if group.is_formal_group?
+    self.volume = user.default_membership_volume if id.nil? && group.is_formal_group?
   end
 end

app/models/outcome.rb

@@ -5,6 +5,7 @@ class Outcome < ApplicationRecord
   include Reactable
   include Translatable
   include HasCreatedEvent
+  include HasEvents
 
   set_custom_fields :calendar_invite, :event_summary, :event_description, :event_location
 

app/models/user.rb

@@ -21,7 +21,7 @@ class User < ApplicationRecord
     demo_bot:   ENV['DEMO_BOT_EMAIL'] || 'contact+demo@loomio.org'
   }.freeze
 
-  devise :database_authenticatable, :recoverable, :registerable, :rememberable, :trackable
+  devise :database_authenticatable, :recoverable, :registerable, :rememberable
   attr_accessor :recaptcha
   attr_accessor :restricted
   attr_accessor :token

app/serializers/discussion_reader_serializer.rb

@@ -33,10 +33,6 @@ def discussion_reader_id
     object.id
   end
 
-  def discussion_reader_volume
-    object.volume
-  end
-
   def seen_by_count
     object.discussion.seen_by_count
   end

app/services/group_export_service.rb

@@ -0,0 +1,68 @@
+class GroupExportService
+  RELATIONS = %w[
+    all_users
+    all_events
+    all_notifications
+    all_documents
+    all_reactions
+    memberships
+    membership_requests
+    discussions
+    polls
+    poll_options
+    poll_did_not_votes
+    poll_unsubscriptions
+    outcomes
+    stances
+    stance_choices
+    discussion_readers
+    comments
+  ]
+
+  JSON_PARAMS = { groups: {methods: [:type]},
+                  users:  {except: [:encrypted_password,
+                                    :reset_password_token,
+                                    :email_api_key,
+                                    :reset_password_token,
+                                    :unsubscribe_token] }}.with_indifferent_access.freeze
+
+  def self.export(groups, filename)
+    ids = Hash.new { |hash, key| hash[key] = [] }
+    File.open(filename, 'w') do |file|
+      groups.each do |group|
+        puts_record(group, file, ids)
+        RELATIONS.each do |relation|
+          puts "Exporting: #{relation}"
+          group.send(relation).find_each(batch_size: 20000) do |record|
+            puts_record(record, file, ids)
+          end
+        end
+      end
+    end
+  end
+
+  def self.export_filename_for(group)
+    "tmp/#{DateTime.now.strftime("%Y-%m-%d_%H-%M-%S")}_#{group.name.parameterize}.json"
+  end
+
+  def self.puts_record(record, file, ids)
+    table = record.class.table_name
+    return if ids[table].include?(record.id)
+    ids[table] << record.id
+    file.puts({table: table, record: record.as_json(JSON_PARAMS[table])}.to_json)
+  end
+
+  def self.import(filename)
+    tables = File.open(filename, 'r').map { |line| JSON.parse(line)['table'] }.uniq
+    tables.each do |table|
+      klass = table.classify.constantize
+      existing_ids = klass.pluck(:id)
+      new_records = File.open(filename, 'r').map do |line|
+        data = JSON.parse(line)
+        next unless (data['table'] == table && !existing_ids.include?(data['record']['id']))
+        klass.new(data['record'])
+      end.compact!
+      klass.import(new_records, validate: false)
+    end
+  end
+end

app/services/group_service.rb

@@ -44,6 +44,11 @@ def self.move(group:, parent:, actor:)
     EventBus.broadcast('group_move', group, parent, actor)
   end
 
+  def self.export(group: , actor: )
+    actor.ability.authorize! :show, group
+    GroupExportJob.perform_later(group, actor)
+  end
+
   def self.merge(source:, target:, actor:)
     actor.ability.authorize! :merge, source
     actor.ability.authorize! :merge, target

app/services/user_service.rb

@@ -28,6 +28,8 @@ def self.delete_many_spam(name_fragment)
 
   def self.delete_spam(user)
     # destroyed (cascade delete)
+    raise "no deletey admin plezse" if user.is_admin?
+
     Group.where(creator_id: user.id).destroy_all
     Poll.where(author_id: user.id).destroy_all
     Discussion.where(author_id: user.id).destroy_all

app/views/user_mailer/group_export_ready.html.haml

@@ -0,0 +1 @@
+%p= t :'user_mailer.group_export_ready.body_html', url: @document.file.url

client/angular/components/group_page/group_actions_dropdown/group_actions_dropdown.coffee

@@ -11,6 +11,18 @@ angular.module('loomioApp').directive 'groupActionsDropdown', ->
   replace: true
   controller: ['$scope', ($scope) ->
 
+    $scope.canExportData = ->
+      Session.user().isMemberOf($scope.group)
+
+    $scope.openGroupExportModal = ->
+      ModalService.open 'ConfirmModal', confirm: ->
+        submit: -> Records.groups.export($scope.group.id)
+        text:
+          title:    'group_export_modal.title'
+          helptext: 'group_export_modal.body'
+          submit:   'group_export_modal.submit'
+          flash:    'group_export_modal.flash'
+
     $scope.canAdministerGroup = ->
       AbilityService.canAdministerGroup($scope.group)