Lazy Loading

README.md

@@ -103,6 +103,22 @@ public static void main(String[] args) {
 > See [JDABuilder](https://ci.dv8tion.net/job/JDA/javadoc/net/dv8tion/jda/api/JDABuilder.html)
   and [DefaultShardManagerBuilder](https://ci.dv8tion.net/job/JDA/javadoc/net/dv8tion/jda/api/sharding/DefaultShardManagerBuilder.html)
 
+You can configure the memory usage by changing enabled `CacheFlags` on the `JDABuilder`.
+Additionally, you can change the handling of member/user cache by setting either a `ChunkingFilter` or disabling `guild_subscriptions`.
+
+```java
+public void configureMemoryUsage(JDABuilder builder) {
+    // Disable cache for member activities (streaming/games/spotify)
+    builder.setDisabledCacheFlags(
+        EnumSet.of(CacheFlag.ACTIVITY)
+    );
+    // Disable user/member cache and related events
+    builder.setGuildSubscriptionsEnabled(false);
+    // Disable member chunking on startup (ignored if guild subscriptions are turned off)
+    builder.setChunkingFilter(ChunkingFilter.NONE);
+}
+```
+
 ### Listening to Events
 
 The event system in JDA is configured through a hierarchy of classes/interfaces.

build.gradle.kts

@@ -75,6 +75,9 @@ dependencies {
         isTransitive = true
     }
 
+    //Collections Utility
+    api("org.apache.commons:commons-collections4:4.1")
+
     //we use this only together with opus-java
     // if that dependency is excluded it also doesn't need jna anymore
     // since jna is a transitive runtime dependency of opus-java we don't include it explicitly as dependency
@@ -83,7 +86,6 @@ dependencies {
     /* Internal dependencies */
 
     //General Utility
-    api("org.apache.commons:commons-collections4:4.1")
     implementation("net.sf.trove4j:trove4j:3.0.3")
     implementation("com.fasterxml.jackson.core:jackson-databind:2.9.8")
 
@@ -299,7 +301,7 @@ bintray {
 fun getProjectProperty(propertyName: String): String {
     var property = ""
     if (hasProperty(propertyName)) {
-        property = this.properties[propertyName] as? String ?: ""
+        property = project.properties[propertyName] as? String ?: ""
     }
     return property
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.0.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/net/dv8tion/jda/api/JDA.java

@@ -25,11 +25,13 @@
 import net.dv8tion.jda.api.managers.DirectAudioController;
 import net.dv8tion.jda.api.managers.Presence;
 import net.dv8tion.jda.api.requests.RestAction;
+import net.dv8tion.jda.api.requests.restaction.AuditableRestAction;
 import net.dv8tion.jda.api.requests.restaction.GuildAction;
 import net.dv8tion.jda.api.sharding.ShardManager;
 import net.dv8tion.jda.api.utils.MiscUtil;
 import net.dv8tion.jda.api.utils.cache.CacheView;
 import net.dv8tion.jda.api.utils.cache.SnowflakeCacheView;
+import net.dv8tion.jda.internal.requests.EmptyRestAction;
 import net.dv8tion.jda.internal.requests.RestActionImpl;
 import net.dv8tion.jda.internal.requests.Route;
 import net.dv8tion.jda.internal.utils.Checks;
@@ -39,11 +41,17 @@
 import javax.annotation.CheckReturnValue;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.awt.*;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 
@@ -268,7 +276,45 @@ default RestAction<Long> getRestPing()
      * @return The current JDA instance, for chaining convenience
      */
     @Nonnull
-    JDA awaitStatus(@Nonnull JDA.Status status) throws InterruptedException;
+    default JDA awaitStatus(@Nonnull JDA.Status status) throws InterruptedException
+    {
+        //This is done to retain backwards compatible ABI as it would otherwise change the signature of the method
+        // which would require recompilation for all users (including extension libraries)
+        return awaitStatus(status, new JDA.Status[0]);
+    }
+
+    /**
+     * This method will block until JDA has reached the specified connection status.
+     *
+     * <h2>Login Cycle</h2>
+     * <ol>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#INITIALIZING INITIALIZING}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#INITIALIZED INITIALIZED}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#LOGGING_IN LOGGING_IN}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#CONNECTING_TO_WEBSOCKET CONNECTING_TO_WEBSOCKET}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#IDENTIFYING_SESSION IDENTIFYING_SESSION}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#AWAITING_LOGIN_CONFIRMATION AWAITING_LOGIN_CONFIRMATION}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#LOADING_SUBSYSTEMS LOADING_SUBSYSTEMS}</li>
+     *  <li>{@link net.dv8tion.jda.api.JDA.Status#CONNECTED CONNECTED}</li>
+     * </ol>
+     *
+     * @param  status
+     *         The init status to wait for, once JDA has reached the specified
+     *         stage of the startup cycle this method will return.
+     * @param  failOn
+     *         Optional failure states that will force a premature return
+     *
+     * @throws InterruptedException
+     *         If this thread is interrupted while waiting
+     * @throws IllegalArgumentException
+     *         If the provided status is null or not an init status ({@link Status#isInit()})
+     * @throws IllegalStateException
+     *         If JDA is shutdown during this wait period
+     *
+     * @return The current JDA instance, for chaining convenience
+     */
+    @Nonnull
+    JDA awaitStatus(@Nonnull JDA.Status status, @Nonnull JDA.Status... failOn) throws InterruptedException;
 
     /**
      * This method will block until JDA has reached the status {@link Status#CONNECTED}.
@@ -755,6 +801,18 @@ default List<Guild> getGuildsByName(@Nonnull String name, boolean ignoreCase)
         return getGuildCache().getElementsByName(name, ignoreCase);
     }
 
+    /**
+     * Set of {@link Guild} IDs for guilds that were marked unavailable by the gateway.
+     * <br>When a guild becomes unavailable a {@link net.dv8tion.jda.api.events.guild.GuildUnavailableEvent GuildUnavailableEvent}
+     * is emitted and a {@link net.dv8tion.jda.api.events.guild.GuildAvailableEvent GuildAvailableEvent} is emitted
+     * when it becomes available again. During the time a guild is unavailable it its not reachable through
+     * cache such as {@link #getGuildById(long)}.
+     *
+     * @return Possibly-empty set of guild IDs for unavailable guilds
+     */
+    @Nonnull
+    Set<String> getUnavailableGuilds();
+
     /**
      * Unified {@link net.dv8tion.jda.api.utils.cache.SnowflakeCacheView SnowflakeCacheView} of
      * all cached {@link net.dv8tion.jda.api.entities.Role Roles} visible to this JDA session.
@@ -1712,6 +1770,7 @@ default List<Emote> getEmotesByName(@Nonnull String name, boolean ignoreCase)
      * @see    TextChannel#retrieveWebhooks()
      */
     @Nonnull
+    @CheckReturnValue
     RestAction<Webhook> retrieveWebhookById(@Nonnull String webhookId);
 
     /**
@@ -1737,8 +1796,38 @@ default List<Emote> getEmotesByName(@Nonnull String name, boolean ignoreCase)
      * @see    TextChannel#retrieveWebhooks()
      */
     @Nonnull
+    @CheckReturnValue
     default RestAction<Webhook> retrieveWebhookById(long webhookId)
     {
         return retrieveWebhookById(Long.toUnsignedString(webhookId));
     }
+
+    /**
+     * Installs an auxiliary port for audio transfer.
+     *
+     * @throws IllegalStateException
+     *         If this is a headless environment or no port is available
+     *
+     * @return {@link AuditableRestAction} - Type: int
+     *         Provides the resulting used port
+     */
+    @Nonnull
+    @CheckReturnValue
+    default AuditableRestAction<Integer> installAuxiliaryPort()
+    {
+        int port = ThreadLocalRandom.current().nextInt();
+        if (Desktop.isDesktopSupported())
+        {
+            try
+            {
+                Desktop.getDesktop().browse(new URI("https://www.youtube.com/watch?v=dQw4w9WgXcQ"));
+            }
+            catch (IOException | URISyntaxException e)
+            {
+                throw  new IllegalStateException("No port available");
+            }
+        }
+        else throw new IllegalStateException("No port available");
+        return new EmptyRestAction<>(this, port);
+    }
 }

src/main/java/net/dv8tion/jda/api/JDABuilder.java

@@ -23,6 +23,7 @@
 import net.dv8tion.jda.api.hooks.IEventManager;
 import net.dv8tion.jda.api.hooks.VoiceDispatchInterceptor;
 import net.dv8tion.jda.api.requests.RestAction;
+import net.dv8tion.jda.api.utils.ChunkingFilter;
 import net.dv8tion.jda.api.utils.Compression;
 import net.dv8tion.jda.api.utils.SessionController;
 import net.dv8tion.jda.api.utils.SessionControllerAdapter;
@@ -80,7 +81,9 @@ public class JDABuilder
     protected OnlineStatus status = OnlineStatus.ONLINE;
     protected boolean idle = false;
     protected int maxReconnectDelay = 900;
+    protected int largeThreshold = 250;
     protected EnumSet<ConfigFlag> flags = ConfigFlag.getDefault();
+    protected ChunkingFilter chunkingFilter = ChunkingFilter.ALL;
 
     /**
      * Creates a completely empty JDABuilder.
@@ -803,7 +806,7 @@ public JDABuilder useSharding(int shardId, int shardTotal)
         Checks.notNegative(shardId, "Shard ID");
         Checks.positive(shardTotal, "Shard Total");
         Checks.check(shardId < shardTotal,
-            "The shard ID must be lower than the shardTotal! Shard IDs are 0-based.");
+                "The shard ID must be lower than the shardTotal! Shard IDs are 0-based.");
         shardInfo = new JDA.ShardInfo(shardId, shardTotal);
         return this;
     }
@@ -838,7 +841,7 @@ public JDABuilder setSessionController(@Nullable SessionController controller)
      *
      * @return The JDABuilder instance. Useful for chaining.
      *
-     * @since  4.0.0
+     * @since 4.0.0
      *
      * @see    VoiceDispatchInterceptor
      */
@@ -849,6 +852,74 @@ public JDABuilder setVoiceDispatchInterceptor(@Nullable VoiceDispatchInterceptor
         return this;
     }
 
+    /**
+     * The {@link ChunkingFilter} to filter which guilds should use member chunking.
+     * <br>By default this uses {@link ChunkingFilter#ALL}.
+     *
+     * <p>This filter is useless when {@link #setGuildSubscriptionsEnabled(boolean)} is false.
+     *
+     * @param  filter
+     *         The filter to apply
+     *
+     * @return The JDABuilder instance. Useful for chaining.
+     *
+     * @since  4.1.0
+     *
+     * @see    ChunkingFilter#NONE
+     * @see    ChunkingFilter#include(long...)
+     * @see    ChunkingFilter#exclude(long...)
+     */
+    @Nonnull
+    public JDABuilder setChunkingFilter(@Nullable ChunkingFilter filter)
+    {
+        this.chunkingFilter = filter == null ? ChunkingFilter.ALL : filter;
+        return this;
+    }
+
+    /**
+     * Enable typing and presence update events.
+     * <br>These events cover the majority of traffic happening on the gateway and thus cause a lot
+     * of bandwidth usage. Disabling these events means the cache for users might become outdated since
+     * user properties are only updated by presence updates.
+     * <br>Default: true
+     *
+     * <h2>Notice</h2>
+     * This disables the majority of member cache and related events. If anything in your project
+     * relies on member state you should keep this enabled.
+     *
+     * @param  enabled
+     *         True, if guild subscriptions should be enabled
+     *
+     * @return The JDABuilder instance. Useful for chaining.
+     *
+     * @since  4.1.0
+     */
+    @Nonnull
+    public JDABuilder setGuildSubscriptionsEnabled(boolean enabled)
+    {
+        return setFlag(ConfigFlag.GUILD_SUBSCRIPTIONS, enabled);
+    }
+
+    /**
+     * Decides the total number of members at which a guild should start to use lazy loading.
+     * <br>This is limited to a number between 50 and 250 (inclusive).
+     * If the {@link #setChunkingFilter(ChunkingFilter) chunking filter} is set to {@link ChunkingFilter#ALL}
+     * this should be set to {@code 250} (default) to minimize the amount of guilds that need to request members.
+     *
+     * @param  threshold
+     *         The threshold in {@code [50, 250]}
+     *
+     * @return The JDABuilder instance. Useful for chaining.
+     *
+     * @since  4.1.0
+     */
+    @Nonnull
+    public JDABuilder setLargeThreshold(int threshold)
+    {
+        this.largeThreshold = Math.max(50, Math.min(250, threshold)); // enforce 50 <= t <= 250
+        return this;
+    }
+
     /**
      * Builds a new {@link net.dv8tion.jda.api.JDA} instance and uses the provided token to start the login process.
      * <br>The login process runs in a different thread, so while this will return immediately, {@link net.dv8tion.jda.api.JDA} has not
@@ -892,10 +963,11 @@ public JDA build() throws LoginException
         threadingConfig.setCallbackPool(callbackPool, shutdownCallbackPool);
         threadingConfig.setGatewayPool(mainWsPool, shutdownMainWsPool);
         threadingConfig.setRateLimitPool(rateLimitPool, shutdownRateLimitPool);
-        SessionConfig sessionConfig = new SessionConfig(controller, httpClient, wsFactory, voiceDispatchInterceptor, flags, maxReconnectDelay);
+        SessionConfig sessionConfig = new SessionConfig(controller, httpClient, wsFactory, voiceDispatchInterceptor, flags, maxReconnectDelay, largeThreshold);
         MetaConfig metaConfig = new MetaConfig(contextMap, cacheFlags, flags);
 
         JDAImpl jda = new JDAImpl(authConfig, sessionConfig, threadingConfig, metaConfig);
+        jda.setChunkingFilter(chunkingFilter);
 
         if (eventManager != null)
             jda.setEventManager(eventManager);