Proxy authentication error

[qifiqi]

Which package is this bug report for? If unsure which one to select, leave blank

@crawlee/cheerio (CheerioCrawler)

Issue description

I slightly modified the demo of the official website and used ProxyConfiguration to set a proxy list. Error 407 was reported, but my authentication information was carried in the url. I am not sure what operation to do

This is the exception log：

/usr/local/bin/node /Users/admin/Desktop/codebaes/crawlee_uitl/req.js
(node:93791) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead.
(Use `node --trace-deprecation ...` to show where the warning was created)
INFO  CheerioCrawler: Starting the crawler.
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":1}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":2}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":3}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":4}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":5}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":6}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":7}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":8}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":9}
WARN  CheerioCrawler: Reclaiming failed request back to the list or queue. Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

 {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","retryCount":10}
ERROR CheerioCrawler: Request failed and reached maximum retries. Error: Detected a session error, rotating session... 
Proxy responded with 407 Proxy Authentication Required: 29 bytes

    at CheerioCrawler._requestFunction (/Users/admin/Desktop/codebaes/crawlee_uitl/node_modules/@crawlee/http/internals/http-crawler.js:448:23)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async wrap (/Users/admin/Desktop/codebaes/crawlee_uitl/node_modules/@apify/timeout/cjs/index.cjs:54:21) {"id":"35Zqyzcm4WOh60y","url":"https://www.zhihu.com/question/603336478/answer/3406593764","method":"GET","uniqueKey":"https://www.zhihu.com/question/603336478/answer/3406593764"}
INFO  CheerioCrawler: All requests from the queue have been processed, the crawler will shut down.
INFO  CheerioCrawler: Final request statistics: {"requestsFinished":0,"requestsFailed":1,"retryHistogram":[null,null,null,null,null,null,null,null,null,null,1],"requestAvgFailedDurationMillis":77,"requestAvgFinishedDurationMillis":null,"requestsFinishedPerMinute":0,"requestsFailedPerMinute":41,"requestTotalDurationMillis":77,"requestsTotal":1,"crawlerRuntimeMillis":1456}
INFO  CheerioCrawler: Error analysis: {"totalErrors":1,"uniqueErrors":1,"mostCommonErrors":["1x: Detected a session error, rotating session... (/Users/admin/Desktop/codebaes/crawlee_uitl/node_modules/@crawlee/http/internals/http-crawler.js:448:23)"]}
INFO  CheerioCrawler: Finished! Total 1 requests: 0 succeeded, 1 failed. {"terminal":true}

进程已结束，退出代码为 0

Code sample

async function getReqCheerio(url, push_redis, proxyUrls) {
    const start_time = Math.ceil(Date.now() / 1000);
    let crawlerOptions = {
        ...opt,
        async requestHandler({ request, response, body, proxyInfo }) {
            // console.log('URL:', request.url);
            console.log('Used proxy:', proxyInfo);
            push_redis({
                url: request.url,
                html: body.toString(),
                status: response.statusCode,
                rt: Math.ceil(Date.now() / 1000) - start_time,
            });
        },

    };

    if (proxyUrls && proxyUrls.length > 0) {
        crawlerOptions.proxyConfiguration = new ProxyConfiguration({
            proxyUrls:proxyUrls,
        })
    }

    const crawler = new CheerioCrawler(crawlerOptions);

    await crawler.run([url]);
}


const proxyUrls = ["http://**:**@***.225.30.161:11273"]; 
await getReqCheerio('https://www.zhihu.com/question/603336478/answer/3406593764', console.log, proxyUrls);

Package version

crawlee@3.10.0

Node.js version

v21.1.0

Operating system

macos

Apify platform

• Tick me if you encountered this issue on the Apify platform

I have tested this on the next release

No response

Other context

No response

[qifiqi]

This agent, I only use other programming languages, or other modules is OK, you can disguise my ip, please take a look, thank you😭😭😭😭

[B4nan]

Take a look where? Unless you run this on apify platform, we don't have any place to check outside of what you provide us. The code itself looks good, those errors are almost always about a wrong proxy URL. If it's about auth, my guess is you don't encode the password properly.

[qifiqi]

When I observed the CheerioCrawler bottom layer in the document, got-scraping was used. I directly used got-scraping to test the coding problem you mentioned. No other characters in my username and password were all normal letters. But the error is still 407 failed to verify
This is the code:

import { gotScraping } from 'got-scraping';

let username = "***";
let password = "***";
let proxy_address = "122.225.30.141:12222";

let encoded_username = encodeURIComponent(username);
let encoded_password = encodeURIComponent(password);

let proxy_url = `http://${encoded_username}:${encoded_password}@${proxy_address}`;

gotScraping
    .get({
        url: 'https://cip.cc/',
        proxyUrl: proxy_url,
        // agent: {
        //     http: agent,
        //     https: agent
        // }
    })
    .then(({ body }) => console.log(body));

this is the exception:

/usr/local/bin/node /Users/admin/Desktop/codebaes/crawlee_uitl/proxy_test.js
node:internal/process/promises:289
            triggerUncaughtException(err, true /* fromPromise */);
            ^

RequestError: Proxy responded with 407 Proxy Authentication Required: 29 bytes
    at Request._beforeError (file:///Users/admin/Desktop/codebaes/crawlee_uitl/node_modules/got/dist/source/core/index.js:310:21)
    at Request.flush (file:///Users/admin/Desktop/codebaes/crawlee_uitl/node_modules/got/dist/source/core/index.js:299:18)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at ClientRequest.<anonymous> (file:///Users/admin/Desktop/codebaes/crawlee_uitl/node_modules/got-scraping/dist/index.js:229:18)
    at Object.onceWrapper (node:events:630:26)
    at ClientRequest.emit (node:events:515:28)
    at Socket.socketOnData (node:_http_client:575:11)
    at Socket.emit (node:events:515:28)
    at addChunk (node:internal/streams/readable:545:12)
    at readableAddChunkPushByteMode (node:internal/streams/readable:495:3)
    at Readable.push (node:internal/streams/readable:375:5)
    at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
  input: undefined,
  code: 'ERR_GOT_REQUEST_ERROR',
  timings: undefined,
  options: Options {
    _unixOptions: { insecureHTTPParser: true },
    _internals: {
      request: [Function (anonymous)],
      agent: {
        http: TransformHeadersAgent {
          agent: HttpRegularProxyAgent {
            _events: [Object: null prototype],
            _eventsCount: 2,
            _maxListeners: undefined,
            defaultPort: 80,
            protocol: 'http:',
            options: [Object: null prototype],
            requests: [Object: null prototype] {},
            sockets: [Object: null prototype] {},
            freeSockets: [Object: null prototype] {},
            keepAliveMsecs: 1000,
            keepAlive: false,
            maxSockets: Infinity,
            maxFreeSockets: 256,
            scheduling: 'lifo',
            maxTotalSockets: Infinity,
            totalSocketCount: 0,
            proxy: URL {},
            [Symbol(kCapture)]: false
          }
        },
        https: TransformHeadersAgent {
          agent: HttpsProxyAgent {
            _events: [Object: null prototype],
            _eventsCount: 2,
            _maxListeners: undefined,
            defaultPort: 443,
            protocol: 'https:',
            options: [Object: null prototype],
            requests: [Object: null prototype] {},
            sockets: [Object: null prototype] {},
            freeSockets: [Object: null prototype] {},
            keepAliveMsecs: 1000,
            keepAlive: false,
            maxSockets: Infinity,
            maxFreeSockets: 256,
            scheduling: 'lifo',
            maxTotalSockets: Infinity,
            totalSocketCount: 0,
            maxCachedSessions: 100,
            _sessionCache: [Object],
            proxy: URL {},
            [Symbol(kCapture)]: false
          }
        },
        http2: Http2OverHttp {
          _events: [Object: null prototype] {},
          _eventsCount: 0,
          _maxListeners: undefined,
          sessions: {},
          queue: {},
          timeout: 0,
          maxSessions: Infinity,
          maxEmptySessions: 0,
          _emptySessionCount: 0,
          _sessionCount: 0,
          settings: { enablePush: false, initialWindowSize: 33554432 },
          tlsSessionCache: QuickLRU {
            maxSize: 100,
            onEviction: undefined,
            cache: Map(0) {},
            oldCache: Map(0) {},
            _size: 0
          },
          proxyOptions: {
            raw: true,
            url: URL {},
            rejectUnauthorized: false,
            headers: {}
          },
          [Symbol(kCapture)]: false
        }
      },
      h2session: undefined,
      decompress: true,
      timeout: {
        connect: undefined,
        lookup: undefined,
        read: undefined,
        request: 60000,
        response: undefined,
        secureConnect: undefined,
        send: undefined,
        socket: undefined
      },
      prefixUrl: '',
      body: undefined,
      form: undefined,
      json: undefined,
      cookieJar: undefined,
      ignoreInvalidCookies: false,
      searchParams: undefined,
      dnsLookup: undefined,
      dnsCache: undefined,
      context: {
        headerGenerator: HeaderGenerator {
          globalOptions: {
            browsers: [Array],
            operatingSystems: [Array],
            devices: [Array],
            locales: [Array],
            httpVersion: '2',
            browserListQuery: '',
            strict: false
          },
          browserListQuery: undefined,
          inputGeneratorNetwork: BayesianNetwork {
            nodesInSamplingOrder: [Array],
            nodesByName: [Object]
          },
          headerGeneratorNetwork: BayesianNetwork {
            nodesInSamplingOrder: [Array],
            nodesByName: [Object]
          },
          uniqueBrowsers: [
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object], [Object], [Object],
            [Object], [Object], [Object], [Object],
            ... 119 more items
          ],
          headersOrder: {
            safari: [Array],
            chrome: [Array],
            firefox: [Array],
            edge: []
          },
          relaxationOrder: [
            'locales',
            'devices',
            'operatingSystems',
            'browsers',
            'browserListQuery'
          ]
        },
        useHeaderGenerator: true,
        insecureHTTPParser: true,
        proxyUrl: 'http://ALBB:UKhAH6hS@122.225.30.141:12222',
        sessionData: undefined
      },
      hooks: {
        init: [
          [Function: optionsValidationHandler],
          [Function: customOptionsHook]
        ],
        beforeRequest: [
          [Function: insecureParserHook],
          [Function: sessionDataHook],
          [Function: http2Hook],
          [AsyncFunction: proxyHook],
          [AsyncFunction: browserHeadersHook],
          [Function: tlsHook]
        ],
        beforeError: [],
        beforeRedirect: [ [Function: refererHook] ],
        beforeRetry: [],
        afterResponse: []
      },
      followRedirect: true,
      maxRedirects: 10,
      cache: undefined,
      throwHttpErrors: false,
      username: '',
      password: '',
      http2: true,
      allowGetBody: false,
      headers: { 'accept-encoding': 'gzip, deflate, br' },
      methodRewriting: false,
      dnsLookupIpVersion: undefined,
      parseJson: [Function: parse],
      stringifyJson: [Function: stringify],
      retry: {
        limit: 0,
        methods: [ 'GET', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'TRACE' ],
        statusCodes: [
          408, 413, 429, 500,
          502, 503, 504, 521,
          522, 524
        ],
        errorCodes: [
          'ETIMEDOUT',
          'ECONNRESET',
          'EADDRINUSE',
          'ECONNREFUSED',
          'EPIPE',
          'ENOTFOUND',
          'ENETUNREACH',
          'EAI_AGAIN'
        ],
        maxRetryAfter: undefined,
        calculateDelay: [Function: calculateDelay],
        backoffLimit: Infinity,
        noise: 100
      },
      localAddress: undefined,
      method: 'GET',
      createConnection: undefined,
      cacheOptions: {
        shared: undefined,
        cacheHeuristic: undefined,
        immutableMinTimeToLive: undefined,
        ignoreCargoCult: undefined
      },
      https: {
        alpnProtocols: undefined,
        rejectUnauthorized: false,
        checkServerIdentity: undefined,
        certificateAuthority: undefined,
        key: undefined,
        certificate: undefined,
        passphrase: undefined,
        pfx: undefined,
        ciphers: undefined,
        honorCipherOrder: undefined,
        minVersion: undefined,
        maxVersion: undefined,
        signatureAlgorithms: undefined,
        tlsSessionLifetime: undefined,
        dhparam: undefined,
        ecdhCurve: undefined,
        certificateRevocationLists: undefined
      },
      encoding: undefined,
      resolveBodyOnly: false,
      isStream: false,
      responseType: 'text',
      url: URL {},
      pagination: {
        transform: [Function: transform],
        paginate: [Function: paginate],
        filter: [Function: filter],
        shouldContinue: [Function: shouldContinue],
        countLimit: Infinity,
        backoff: 0,
        requestLimit: 10000,
        stackAllItems: false
      },
      setHost: true,
      maxHeaderSize: undefined,
      signal: undefined,
      enableUnixSockets: false
    },
    _merging: false,
    _init: [
      {
        handlers: [ [Function: fixDecompress] ],
        mutableDefaults: true,
        http2: true,
        https: { rejectUnauthorized: false },
        throwHttpErrors: false,
        timeout: { request: 60000 },
        retry: { limit: 0 },
        headers: { 'user-agent': undefined },
        context: {
          headerGenerator: HeaderGenerator {
            globalOptions: [Object],
            browserListQuery: undefined,
            inputGeneratorNetwork: [BayesianNetwork],
            headerGeneratorNetwork: [BayesianNetwork],
            uniqueBrowsers: [Array],
            headersOrder: [Object],
            relaxationOrder: [Array]
          },
          useHeaderGenerator: true,
          insecureHTTPParser: true
        },
        agent: {
          http: TransformHeadersAgent { agent: [Agent] },
          https: TransformHeadersAgent { agent: [Agent] }
        },
        hooks: {
          init: [
            [Function: optionsValidationHandler],
            [Function: customOptionsHook]
          ],
          beforeRequest: [
            [Function: insecureParserHook],
            [Function: sessionDataHook],
            [Function: http2Hook],
            [AsyncFunction: proxyHook],
            [AsyncFunction: browserHeadersHook],
            [Function: tlsHook]
          ],
          beforeRedirect: [ [Function: refererHook] ]
        }
      },
      { method: 'get' }
    ]
  }
}

Node.js v21.1.0

进程已结束，退出代码为 1

[qifiqi]

The agent I used is the agent I purchased by myself, I am not sure whether it is not supported

[barjin]

We've added logging for the actual proxy error in got-scraping@4.0.6 - try updating your got-scraping version to 4.0.6 by running npm i got-scraping@latest and running your script again.

With the new package version, you should see the parsed response in the error, which should give you a better idea about what is happening.

[qifiqi]

ok I'll update and try to catch an exception