GitHub rate limit exceed when token provided

[sadortun]

When a Github token is provided, and valid, BUT the rate limit is exceed. Composer ask to create a new token, which is very miscleading.

How to reproduce:

1. Perform 5000 API requests to Github within one hour
2. Verify that https://api.github.com/rate_limit?access_token=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX "remaining": 0,
3. Run composer update

My composer.json:

{
    "name": "symfony/symfony",
    "type": "library",
    "description": "The Symfony PHP framework",
    "keywords": ["framework"],
    "homepage": "https://symfony.com",
    "license": "MIT",
    "authors": [
        {
            "name": "Fabien Potencier",
            "email": "fabien@symfony.com"
        },
        {
            "name": "Symfony Community",
            "homepage": "https://symfony.com/contributors"
        }
    ],
    "require": {
        "php": "^7.1.3",
        "ext-xml": "*",
        "doctrine/common": "~2.4",
        "fig/link-util": "^1.0",
        "twig/twig": "~1.34|~2.4",
        "psr/cache": "~1.0",
        "psr/container": "^1.0",
        "psr/link": "^1.0",
        "psr/log": "~1.0",
        "psr/simple-cache": "^1.0",
        "symfony/polyfill-intl-icu": "~1.0",
        "symfony/polyfill-mbstring": "~1.0"
    },
    "replace": {
        "symfony/asset": "self.version",
        "symfony/browser-kit": "self.version",
        "symfony/cache": "self.version",
        "symfony/config": "self.version",
        "symfony/console": "self.version",
        "symfony/css-selector": "self.version",
        "symfony/dependency-injection": "self.version",
        "symfony/debug": "self.version",
        "symfony/debug-bundle": "self.version",
        "symfony/doctrine-bridge": "self.version",
        "symfony/dom-crawler": "self.version",
        "symfony/dotenv": "self.version",
        "symfony/event-dispatcher": "self.version",
        "symfony/expression-language": "self.version",
        "symfony/filesystem": "self.version",
        "symfony/finder": "self.version",
        "symfony/form": "self.version",
        "symfony/framework-bundle": "self.version",
        "symfony/http-foundation": "self.version",
        "symfony/http-kernel": "self.version",
        "symfony/inflector": "self.version",
        "symfony/intl": "self.version",
        "symfony/ldap": "self.version",
        "symfony/lock": "self.version",
        "symfony/monolog-bridge": "self.version",
        "symfony/options-resolver": "self.version",
        "symfony/process": "self.version",
        "symfony/property-access": "self.version",
        "symfony/property-info": "self.version",
        "symfony/proxy-manager-bridge": "self.version",
        "symfony/routing": "self.version",
        "symfony/security": "self.version",
        "symfony/security-core": "self.version",
        "symfony/security-csrf": "self.version",
        "symfony/security-guard": "self.version",
        "symfony/security-http": "self.version",
        "symfony/security-bundle": "self.version",
        "symfony/serializer": "self.version",
        "symfony/stopwatch": "self.version",
        "symfony/templating": "self.version",
        "symfony/translation": "self.version",
        "symfony/twig-bridge": "self.version",
        "symfony/twig-bundle": "self.version",
        "symfony/validator": "self.version",
        "symfony/var-dumper": "self.version",
        "symfony/web-link": "self.version",
        "symfony/web-profiler-bundle": "self.version",
        "symfony/web-server-bundle": "self.version",
        "symfony/workflow": "self.version",
        "symfony/yaml": "self.version"
    },
    "require-dev": {
        "cache/integration-tests": "dev-master",
        "doctrine/cache": "~1.6",
        "doctrine/data-fixtures": "1.0.*",
        "doctrine/dbal": "~2.4",
        "doctrine/orm": "~2.4,>=2.4.5",
        "doctrine/doctrine-bundle": "~1.4",
        "monolog/monolog": "~1.11",
        "ocramius/proxy-manager": "~0.4|~1.0|~2.0",
        "predis/predis": "~1.0",
        "egulias/email-validator": "~1.2,>=1.2.8|~2.0",
        "symfony/phpunit-bridge": "~3.2",
        "symfony/polyfill-apcu": "~1.1",
        "symfony/security-acl": "~2.8|~3.0",
        "phpdocumentor/reflection-docblock": "^3.0|^4.0"
    },
    "conflict": {
        "phpdocumentor/reflection-docblock": "<3.0||>=3.2.0,<3.2.2",
        "phpdocumentor/type-resolver": "<0.2.0",
        "phpunit/phpunit": "<4.8.35|<5.4.3,>=5.0"
    },
    "provide": {
        "psr/cache-implementation": "1.0",
        "psr/container-implementation": "1.0",
        "psr/simple-cache-implementation": "1.0"
    },
    "autoload": {
        "psr-4": {
            "Symfony\\Bridge\\Doctrine\\": "src/Symfony/Bridge/Doctrine/",
            "Symfony\\Bridge\\Monolog\\": "src/Symfony/Bridge/Monolog/",
            "Symfony\\Bridge\\ProxyManager\\": "src/Symfony/Bridge/ProxyManager/",
            "Symfony\\Bridge\\Twig\\": "src/Symfony/Bridge/Twig/",
            "Symfony\\Bundle\\": "src/Symfony/Bundle/",
            "Symfony\\Component\\": "src/Symfony/Component/"
        },
        "classmap": [
            "src/Symfony/Component/Intl/Resources/stubs"
        ],
        "exclude-from-classmap": [
            "**/Tests/"
        ]
    },
    "autoload-dev": {
        "files": [ "src/Symfony/Component/VarDumper/Resources/functions/dump.php" ]
    },
    "minimum-stability": "dev",
    "extra": {
        "branch-alias": {
            "dev-master": "4.0-dev"
        }
    }
}

Output of composer diagnose:

bash-4.2$ composer diag
Checking composer.json: WARNING
Defining autoload.psr-4 with an empty namespace prefix is a bad idea for performance
emantic versioning
Checking platform settings: OK
Checking git settings: OK
Checking http connectivity to packagist: OK
Checking https connectivity to packagist: OK
Checking github.com oauth access: FAIL
[Composer\Downloader\TransportException] The "https://api.github.com/" file could not be downloaded (HTTP/1.1 403 Forbidden)
Checking disk free space: OK
Checking pubkeys:
Tags Public Key Fingerprint: 57815BA2 7E54DC31 7ECC7CC5 573090D0  87719BA6 8F3BB723 4E5D42D0 84A14642
Dev Public Key Fingerprint: 4AC45767 E5EC2265 2F0C1167 CBBB8A2B  0C708369 153E328C AD90147D AFE50952
OK
Checking composer version: OK

When I run this command:

composer update

I get the following output:

Downloading https://api.github.com/repos/jquery/jquery-dist

Could not fetch https://api.github.com/repos/jquery/jquery-dist, please create a GitHub OAuth token to go over the API rate limit
Head to https://github.com/settings/tokens/new?scopes=repo&description=Composer+on+ns509029.ip-192-95-32.net+2017-08-21+2151
to retrieve a token. It will be stored in "/var/www/vhosts/newera.systems/.composer/auth.json" for future use by Composer.

And I expected this to happen:

When veryfing the RATE limit manually,
GET https://api.github.com/rate_limit?access_token=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

{
 "rate": {
    "limit": 5000,
    "remaining": 0,
    "reset": 1503354368
  }

The following output would be expected :

Downloading https://api.github.com/repos/jquery/jquery-dist

Could not fetch https://api.github.com/repos/jquery/jquery-dist,  The current  GitHub OAuth token exceed the. API rate limit. Please wait until 2017-04-21  18:33:21  and retry the request.

[alcohol]

I think that indeed we not check for this. It is incredibly rare for someone to hit the rate limit associated with a token. It would be a nice-to-have to handle this more gracefully.

[sadortun]

@alcohol Yes, i was very suprised of this also, the project have many npm-assets and it look like a single composer update require 2600 requests to the API .... ouch ... !

[alcohol]

Yeah I personally cannot recommend managing npm dependencies through composer.. but opinions vary :-)

[curry684]

@alcohol we do, the full error message is:

Could not fetch https://api.github.com/repos/jquery/jquery-dist, please create a GitHub OAuth token to go over the API rate limit

It dates back from before the anonymous API limits were dramatically increased, but the error is technically still completely correct: it says the rate limit is the problem, and how to exceed it.

I don't see the problem.

[hkdobrev]

@curry684 The problem is two-fold:

1. Composer error message is like no OAuth token is used even when you are using one, but the extended rate limit is exceeded.
2. Composer doesn't read the response from GitHub to understand the extended rate limit with a token is exceeded and to report back to the user accordingly.

Creating another token at that time would suffer from the same rate limit.

[curry684]

Ow I see, my apologies. True issue then, but small :P

[Deamon]

Having a look at the code, it seems that on master branch it is already managed :

https://github.com/composer/composer/blob/master/src/Composer/Repository/Vcs/GitHubDriver.php#L397-L404

is it an other case ?

[curry684]

That code is 4 years and a week old so I doubt it's still working as it should, feel free to make a PR :)

[hkdobrev]

Everything in that code seems correct according to https://developer.github.com/v3/#rate-limiting (headers, status code etc. are still the same)
However, you need to exceed the rate-limit of 5,000 in order to debug it 😆

[Deamon]

I checked on this link before submiting https://developer.github.com/v3/rate_limit/ and the code still correct.
Not easy to debug this one.

[Seldaek]

Most likely the bug is in

composer/src/Composer/Util/RemoteFilesystem.php

Lines 581 to 588 in eb61069

	if ($this->config && in_array($this->originUrl, $this->config->get('github-domains'), true)) {
	$message = "\n".'Could not fetch '.$this->fileUrl.', please create a GitHub OAuth token '.($httpStatus === 404 ? 'to access private repos' : 'to go over the API rate limit');
	$gitHubUtil = new GitHub($this->io, $this->config, null);
	if (!$gitHubUtil->authorizeOAuth($this->originUrl)
	&& (!$this->io->isInteractive() || !$gitHubUtil->authorizeOAuthInteractively($this->originUrl, $message))
	) {
	throw new TransportException('Could not authenticate against '.$this->originUrl, 401);
	}

not doing the same check as GitHubDriver does.

Perhaps GitHubDriver::getRateLimit should be moved to Util\GitHub so that this can be called from both places.

[dmoagx]

I was running into a variant of this issue which took an annoyingly large amount of caveman debugging to find:

Composer 1.7.1 was reporting the "please create a GitHub OAuth token to go over the API rate limit" error at the very first download from GitHub, which pointed me in the completely wrong direction (since it doesn't make sense to hit a rate limit on the first download).

Anyway, only after dumping everything that goes in and out of the getRemoteContents method it became clear what happened:

========== $http_response_header ============================================================================
array(21) {
  [0]=>
  string(25) "HTTP/1.1 401 Unauthorized"
  [1]=>
  string(18) "Server: GitHub.com"
  [2]=>
  string(35) "Date: Thu, 09 Aug 2018 21:26:46 GMT"
  [3]=>
  string(45) "Content-Type: application/json; charset=utf-8"
  [4]=>
  string(18) "Content-Length: 83"
  [5]=>
  string(17) "Connection: close"
  [6]=>
  string(24) "Status: 401 Unauthorized"
  [7]=>
  string(43) "X-GitHub-Media-Type: github.v3; format=json"
  [8]=>
  string(21) "X-RateLimit-Limit: 60"
  [9]=>
  string(25) "X-RateLimit-Remaining: 58"
  [10]=>
  string(29) "X-RateLimit-Reset: 1533853522"
  [11]=>
  string(187) "Access-Control-Expose-Headers: ETag, Link, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval"
  [12]=>
  string(30) "Access-Control-Allow-Origin: *"
  [13]=>
  string(71) "Strict-Transport-Security: max-age=31536000; includeSubdomains; preload"
  [14]=>
  string(21) "X-Frame-Options: deny"
  [15]=>
  string(31) "X-Content-Type-Options: nosniff"
  [16]=>
  string(31) "X-XSS-Protection: 1; mode=block"
  [17]=>
  string(74) "Referrer-Policy: origin-when-cross-origin, strict-origin-when-cross-origin"
  [18]=>
  string(43) "Content-Security-Policy: default-src 'none'"
  [19]=>
  string(24) "X-Runtime-rack: 0.019195"
  [20]=>
  string(54) "X-GitHub-Request-Id: E591:420D:BF4E99:15BE7C6:5B6CB196"
}
========== $originUrl ============================================================================
string(10) "github.com"
========== $fileUrl ============================================================================
string(154) "https://api.github.com/repos/symfony/polyfill-ctype/zipball/e3d826245268269cd66f8326bd8bc066687b4a19?access_token=...removed..."
========== $result ============================================================================
string(83) "{"message":"Bad credentials","documentation_url":"https://developer.github.com/v3"}"

Could not fetch https://api.github.com/repos/symfony/polyfill-ctype/zipball/e3d826245268269cd66f8326bd8bc066687b4a19, please create a GitHub OAuth token to go over the API rate limit

So GitHub even includes headers in their response now that clearly state that rate limit is not the issue here...

Only the real error message "Bad credentials" together with the fact that the actual URL (while composer only shows a truncated variant) already had an access token appended made me realize, that I already had an OAuth token configured multiple years ago, which has become invalid in the meantime.

Simply deleting it fixed the error, but the message is highly misleading (and the fact that composer -vvv does still not print the http headers did not contribute).