How to use ForgetRequest method of PuppeteerSharp.NetworkManager class

Best Puppeteer-sharp code snippet using PuppeteerSharp.NetworkManager.ForgetRequest

Run Puppeteer-sharp automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

NetworkManager.cs

Source: NetworkManager.cs Github

copy
1using System;
2using System.Collections.Generic;
3using System.Globalization;
4using System.Threading.Tasks;
5using Microsoft.Extensions.Logging;
6using PuppeteerSharp.Helpers;
7using PuppeteerSharp.Helpers.Json;
8using PuppeteerSharp.Messaging;
9
10namespace PuppeteerSharp
11{
12    internal class NetworkManager
13    {
14        private readonly CDPSession _client;
15        private readonly ILogger _logger;
16        private readonly ConcurrentSet<string> _attemptedAuthentications = new();
17        private readonly bool _ignoreHTTPSErrors;
18        private readonly InternalNetworkConditions _emulatedNetworkConditions = new()
19        {
20            Offline = false,
21            Upload = -1,
22            Download = -1,
23            Latency = 0,
24        };
25
26        private readonly NetworkEventManager _networkEventManager = new();
27
28        private Dictionary<string, string> _extraHTTPHeaders;
29        private Credentials _credentials;
30        private bool _userRequestInterceptionEnabled;
31        private bool _userCacheDisabled;
32        private bool _protocolRequestInterceptionEnabled;
33
34        internal NetworkManager(CDPSession client, bool ignoreHTTPSErrors, FrameManager frameManager)
35        {
36            FrameManager = frameManager;
37            _client = client;
38            _ignoreHTTPSErrors = ignoreHTTPSErrors;
39            _client.MessageReceived += Client_MessageReceived;
40            _logger = _client.Connection.LoggerFactory.CreateLogger<NetworkManager>();
41        }
42
43        internal event EventHandler<ResponseCreatedEventArgs> Response;
44
45        internal event EventHandler<RequestEventArgs> Request;
46
47        internal event EventHandler<RequestEventArgs> RequestFinished;
48
49        internal event EventHandler<RequestEventArgs> RequestFailed;
50
51        internal event EventHandler<RequestEventArgs> RequestServedFromCache;
52
53        internal Dictionary<string, string> ExtraHTTPHeaders => _extraHTTPHeaders?.Clone();
54
55        internal FrameManager FrameManager { get; set; }
56
57        internal int NumRequestsInProgress => _networkEventManager.NumRequestsInProgress;
58
59        internal async Task InitializeAsync()
60        {
61            await _client.SendAsync("Network.enable").ConfigureAwait(false);
62            if (_ignoreHTTPSErrors)
63            {
64                await _client.SendAsync("Security.setIgnoreCertificateErrors", new SecuritySetIgnoreCertificateErrorsRequest
65                {
66                    Ignore = true
67                }).ConfigureAwait(false);
68            }
69        }
70
71        internal Task AuthenticateAsync(Credentials credentials)
72        {
73            _credentials = credentials;
74            return UpdateProtocolRequestInterceptionAsync();
75        }
76
77        internal Task SetExtraHTTPHeadersAsync(Dictionary<string, string> extraHTTPHeaders)
78        {
79            _extraHTTPHeaders = new Dictionary<string, string>();
80
81            foreach (var item in extraHTTPHeaders)
82            {
83                _extraHTTPHeaders[item.Key.ToLower(CultureInfo.CurrentCulture)] = item.Value;
84            }
85            return _client.SendAsync("Network.setExtraHTTPHeaders", new NetworkSetExtraHTTPHeadersRequest
86            {
87                Headers = _extraHTTPHeaders
88            });
89        }
90
91        internal async Task SetOfflineModeAsync(bool value)
92        {
93            _emulatedNetworkConditions.Offline = value;
94            await UpdateNetworkConditionsAsync().ConfigureAwait(false);
95        }
96
97        internal async Task EmulateNetworkConditionsAsync(NetworkConditions networkConditions)
98        {
99            _emulatedNetworkConditions.Upload = networkConditions?.Upload ?? -1;
100            _emulatedNetworkConditions.Download = networkConditions?.Download ?? -1;
101            _emulatedNetworkConditions.Latency = networkConditions?.Latency ?? 0;
102            await UpdateNetworkConditionsAsync().ConfigureAwait(false);
103        }
104
105        private Task UpdateNetworkConditionsAsync()
106            => _client.SendAsync("Network.emulateNetworkConditions", new NetworkEmulateNetworkConditionsRequest
107            {
108                Offline = _emulatedNetworkConditions.Offline,
109                Latency = _emulatedNetworkConditions.Latency,
110                UploadThroughput = _emulatedNetworkConditions.Upload,
111                DownloadThroughput = _emulatedNetworkConditions.Download,
112            });
113
114        internal Task SetUserAgentAsync(string userAgent)
115            => _client.SendAsync("Network.setUserAgentOverride", new NetworkSetUserAgentOverrideRequest
116            {
117                UserAgent = userAgent
118            });
119
120        internal Task SetCacheEnabledAsync(bool enabled)
121        {
122            _userCacheDisabled = !enabled;
123            return UpdateProtocolCacheDisabledAsync();
124        }
125
126        internal Task SetRequestInterceptionAsync(bool value)
127        {
128            _userRequestInterceptionEnabled = value;
129            return UpdateProtocolRequestInterceptionAsync();
130        }
131
132        private Task UpdateProtocolCacheDisabledAsync()
133            => _client.SendAsync("Network.setCacheDisabled", new NetworkSetCacheDisabledRequest
134            {
135                CacheDisabled = _userCacheDisabled
136            });
137
138        private async void Client_MessageReceived(object sender, MessageEventArgs e)
139        {
140            try
141            {
142                switch (e.MessageID)
143                {
144                    case "Fetch.requestPaused":
145                        await OnRequestPausedAsync(e.MessageData.ToObject<FetchRequestPausedResponse>(true)).ConfigureAwait(false);
146                        break;
147                    case "Fetch.authRequired":
148                        await OnAuthRequiredAsync(e.MessageData.ToObject<FetchAuthRequiredResponse>(true)).ConfigureAwait(false);
149                        break;
150                    case "Network.requestWillBeSent":
151                        await OnRequestWillBeSentAsync(e.MessageData.ToObject<RequestWillBeSentPayload>(true)).ConfigureAwait(false);
152                        break;
153                    case "Network.requestServedFromCache":
154                        OnRequestServedFromCache(e.MessageData.ToObject<RequestServedFromCacheResponse>(true));
155                        break;
156                    case "Network.responseReceived":
157                        OnResponseReceived(e.MessageData.ToObject<ResponseReceivedResponse>(true));
158                        break;
159                    case "Network.loadingFinished":
160                        OnLoadingFinished(e.MessageData.ToObject<LoadingFinishedEventResponse>(true));
161                        break;
162                    case "Network.loadingFailed":
163                        OnLoadingFailed(e.MessageData.ToObject<LoadingFailedEventResponse>(true));
164                        break;
165                    case "Network.responseReceivedExtraInfo":
166                        await OnResponseReceivedExtraInfoAsync(e.MessageData.ToObject<ResponseReceivedExtraInfoResponse>(true)).ConfigureAwait(false);
167                        break;
168                }
169            }
170            catch (Exception ex)
171            {
172                var message = $"NetworkManager failed to process {e.MessageID}. {ex.Message}. {ex.StackTrace}";
173                _logger.LogError(ex, message);
174                _client.Close(message);
175            }
176        }
177
178        private async Task OnResponseReceivedExtraInfoAsync(ResponseReceivedExtraInfoResponse e)
179        {
180            var redirectInfo = _networkEventManager.TakeQueuedRedirectInfo(e.RequestId);
181
182            if (redirectInfo != null)
183            {
184                _networkEventManager.ResponseExtraInfo(e.RequestId).Add(e);
185                await OnRequestAsync(redirectInfo.Event, redirectInfo.FetchRequestId).ConfigureAwait(false);
186                return;
187            }
188
189            // We may have skipped response and loading events because we didn't have
190            // this ExtraInfo event yet. If so, emit those events now.
191            var queuedEvents = _networkEventManager.GetQueuedEventGroup(e.RequestId);
192
193            if (queuedEvents != null)
194            {
195                EmitResponseEvent(queuedEvents.ResponseReceivedEvent, e);
196
197                if (queuedEvents.LoadingFinishedEvent != null) {
198                    OnLoadingFinished(queuedEvents.LoadingFinishedEvent);
199                }
200
201                if (queuedEvents.LoadingFailedEvent != null) {
202                    OnLoadingFailed(queuedEvents.LoadingFailedEvent);
203                }
204
205                // We need this in .NET to avoid race conditions
206                _networkEventManager.ForgetQueuedEventGroup(e.RequestId);
207                return;
208            }
209
210            // Wait until we get another event that can use this ExtraInfo event.
211            _networkEventManager.ResponseExtraInfo(e.RequestId).Add(e);
212        }
213
214        private void OnLoadingFailed(LoadingFailedEventResponse e)
215        {
216            var queuedEvents = _networkEventManager.GetQueuedEventGroup(e.RequestId);
217
218            if (queuedEvents != null)
219            {
220                queuedEvents.LoadingFailedEvent = e;
221            }
222            else
223            {
224                EmitLoadingFailed(e);
225            }
226        }
227
228        private void EmitLoadingFailed(LoadingFailedEventResponse e)
229        {
230            var request = _networkEventManager.GetRequest(e.RequestId);
231            if (request == null)
232            {
233                return;
234            }
235
236            request.Failure = e.ErrorText;
237            request.Response?.BodyLoadedTaskWrapper.TrySetResult(true);
238
239            ForgetRequest(request, true);
240
241            RequestFailed?.Invoke(this, new RequestEventArgs
242            {
243                Request = request
244            });
245        }
246
247        private void OnLoadingFinished(LoadingFinishedEventResponse e)
248        {
249            var queuedEvents = _networkEventManager.GetQueuedEventGroup(e.RequestId);
250
251            if (queuedEvents != null)
252            {
253                queuedEvents.LoadingFinishedEvent = e;
254            }
255            else
256            {
257                EmitLoadingFinished(e);
258            }
259        }
260
261        private void EmitLoadingFinished(LoadingFinishedEventResponse e)
262        {
263            var request = _networkEventManager.GetRequest(e.RequestId);
264            if (request == null)
265            {
266                return;
267            }
268
269            request.Response?.BodyLoadedTaskWrapper.TrySetResult(true);
270
271            ForgetRequest(request, true);
272
273            RequestFinished?.Invoke(this, new RequestEventArgs
274            {
275                Request = request
276            });
277        }
278
279        private void ForgetRequest(Request request, bool events)
280        {
281            _networkEventManager.ForgetRequest(request.RequestId);
282
283            if (request.InterceptionId != null)
284            {
285                _attemptedAuthentications.Remove(request.InterceptionId);
286            }
287
288            if (events)
289            {
290                _networkEventManager.Forget(request.RequestId);
291            }
292        }
293
294        private void OnResponseReceived(ResponseReceivedResponse e)
295        {
296            var request = _networkEventManager.GetRequest(e.RequestId);
297            ResponseReceivedExtraInfoResponse extraInfo = null;
298
299            if (request != null && !request.FromMemoryCache && e.HasExtraInfo)
300            {
301                extraInfo = _networkEventManager.ShiftResponseExtraInfo(e.RequestId);
302
303                if (extraInfo == null)
304                {
305                    _networkEventManager.QueuedEventGroup(e.RequestId, new()
306                    {
307                        ResponseReceivedEvent = e
308                    });
309                    return;
310                }
311            }
312
313            EmitResponseEvent(e, extraInfo);
314        }
315
316        private void EmitResponseEvent(ResponseReceivedResponse e, ResponseReceivedExtraInfoResponse extraInfo)
317        {
318            var request = _networkEventManager.GetRequest(e.RequestId);
319
320            // FileUpload sends a response without a matching request.
321
322            if (request == null)
323            {
324                return;
325            }
326
327            var response = new Response(
328                _client,
329                request,
330                e.Response,
331                extraInfo);
332
333            request.Response = response;
334
335            Response?.Invoke(this, new ResponseCreatedEventArgs
336            {
337                Response = response
338            });
339        }
340
341        private async Task OnAuthRequiredAsync(FetchAuthRequiredResponse e)
342        {
343            var response = "Default";
344            if (_attemptedAuthentications.Contains(e.RequestId))
345            {
346                response = "CancelAuth";
347            }
348            else if (_credentials != null)
349            {
350                response = "ProvideCredentials";
351                _attemptedAuthentications.Add(e.RequestId);
352            }
353            var credentials = _credentials ?? new Credentials();
354            try
355            {
356                await _client.SendAsync("Fetch.continueWithAuth", new ContinueWithAuthRequest
357                {
358                    RequestId = e.RequestId,
359                    AuthChallengeResponse = new ContinueWithAuthRequestChallengeResponse
360                    {
361                        Response = response,
362                        Username = credentials.Username,
363                        Password = credentials.Password
364                    }
365                }).ConfigureAwait(false);
366            }
367            catch (PuppeteerException ex)
368            {
369                _logger.LogError(ex.ToString());
370            }
371        }
372
373        private async Task OnRequestPausedAsync(FetchRequestPausedResponse e)
374        {
375            if (!_userRequestInterceptionEnabled && _protocolRequestInterceptionEnabled)
376            {
377                try
378                {
379                    await _client.SendAsync("Fetch.continueRequest", new FetchContinueRequestRequest
380                    {
381                        RequestId = e.RequestId
382                    }).ConfigureAwait(false);
383                }
384                catch (PuppeteerException ex)
385                {
386                    _logger.LogError(ex.ToString());
387                }
388            }
389
390            if (string.IsNullOrEmpty(e.NetworkId))
391            {
392                return;
393            }
394
395            var requestWillBeSentEvent =
396              _networkEventManager.GetRequestWillBeSent(e.NetworkId);
397
398            // redirect requests have the same `requestId`,
399            if (
400                requestWillBeSentEvent != null &&
401                (requestWillBeSentEvent.Request.Url != e.Request.Url ||
402                requestWillBeSentEvent.Request.Method != e.Request.Method))
403            {
404                _networkEventManager.ForgetRequestWillBeSent(e.NetworkId);
405                requestWillBeSentEvent = null;
406            }
407
408            if (requestWillBeSentEvent != null)
409            {
410                PatchRequestEventHeaders(requestWillBeSentEvent, e);
411                await OnRequestAsync(requestWillBeSentEvent, e.RequestId).ConfigureAwait(false);
412            }
413            else
414            {
415                _networkEventManager.StoreRequestPaused(e.NetworkId, e);
416            }
417        }
418
419        private async Task OnRequestAsync(RequestWillBeSentPayload e, string fetchRequestId)
420        {
421            Request request;
422            var redirectChain = new List<Request>();
423            if (e.RedirectResponse != null)
424            {
425                ResponseReceivedExtraInfoResponse redirectResponseExtraInfo = null;
426                if (e.RedirectHasExtraInfo)
427                {
428                    redirectResponseExtraInfo = _networkEventManager.ShiftResponseExtraInfo(e.RequestId);
429                    if (redirectResponseExtraInfo == null)
430                    {
431                        _networkEventManager.QueueRedirectInfo(e.RequestId, new()
432                        {
433                            Event = e,
434                            FetchRequestId = fetchRequestId,
435                        });
436                        return;
437                    }
438                }
439
440                request = _networkEventManager.GetRequest(e.RequestId);
441
442                // If we connect late to the target, we could have missed the requestWillBeSent event.
443                if (request != null)
444                {
445                    HandleRequestRedirect(request, e.RedirectResponse, redirectResponseExtraInfo);
446                    redirectChain = request.RedirectChainList;
447                }
448            }
449
450            var frame = !string.IsNullOrEmpty(e.FrameId) ? await FrameManager.TryGetFrameAsync(e.FrameId).ConfigureAwait(false) : null;
451
452            request = new Request(
453                _client,
454                frame,
455                fetchRequestId,
456                _userRequestInterceptionEnabled,
457                e,
458                redirectChain);
459
460            _networkEventManager.StoreRequest(e.RequestId, request);
461
462            Request?.Invoke(this, new RequestEventArgs
463            {
464                Request = request
465            });
466        }
467
468        private void OnRequestServedFromCache(RequestServedFromCacheResponse response)
469        {
470            var request = _networkEventManager.GetRequest(response.RequestId);
471
472            if (request != null)
473            {
474                request.FromMemoryCache = true;
475            }
476            RequestServedFromCache?.Invoke(this, new RequestEventArgs { Request = request });
477        }
478
479        private void HandleRequestRedirect(Request request, ResponsePayload responseMessage, ResponseReceivedExtraInfoResponse extraInfo)
480        {
481            var response = new Response(
482                _client,
483                request,
484                responseMessage,
485                extraInfo);
486
487            request.Response = response;
488            request.RedirectChainList.Add(request);
489            response.BodyLoadedTaskWrapper.TrySetException(
490                new PuppeteerException("Response body is unavailable for redirect responses"));
491
492            ForgetRequest(request, false);
493
494            Response?.Invoke(this, new ResponseCreatedEventArgs
495            {
496                Response = response
497            });
498
499            RequestFinished?.Invoke(this, new RequestEventArgs
500            {
501                Request = request
502            });
503        }
504
505        private async Task OnRequestWillBeSentAsync(RequestWillBeSentPayload e)
506        {
507            // Request interception doesn't happen for data URLs with Network Service.
508            if (_userRequestInterceptionEnabled && !e.Request.Url.StartsWith("data:", StringComparison.InvariantCultureIgnoreCase))
509            {
510                _networkEventManager.StoreRequestWillBeSent(e.RequestId, e);
511
512                var requestPausedEvent = _networkEventManager.GetRequestPaused(e.RequestId);
513                if (requestPausedEvent != null)
514                {
515                    var fetchRequestId = requestPausedEvent.RequestId;
516                    PatchRequestEventHeaders(e, requestPausedEvent);
517                    await OnRequestAsync(e, fetchRequestId).ConfigureAwait(false);
518                    _networkEventManager.ForgetRequestPaused(e.RequestId);
519                }
520
521                return;
522            }
523            await OnRequestAsync(e, null).ConfigureAwait(false);
524        }
525
526        private void PatchRequestEventHeaders(RequestWillBeSentPayload requestWillBeSentEvent, FetchRequestPausedResponse requestPausedEvent)
527        {
528            foreach (var kv in requestPausedEvent.Request.Headers)
529            {
530                requestWillBeSentEvent.Request.Headers[kv.Key] = kv.Value;
531            }
532        }
533
534        private async Task UpdateProtocolRequestInterceptionAsync()
535        {
536            var enabled = _userRequestInterceptionEnabled || _credentials != null;
537
538            if (enabled == _protocolRequestInterceptionEnabled)
539            {
540                return;
541            }
542            _protocolRequestInterceptionEnabled = enabled;
543            if (enabled)
544            {
545                await Task.WhenAll(
546                    UpdateProtocolCacheDisabledAsync(),
547                    _client.SendAsync("Fetch.enable", new FetchEnableRequest
548                    {
549                        HandleAuthRequests = true,
550                        Patterns = new[] { new FetchEnableRequest.Pattern { UrlPattern = "*" } }
551                    })).ConfigureAwait(false);
552            }
553            else
554            {
555                await Task.WhenAll(
556                    UpdateProtocolCacheDisabledAsync(),
557                    _client.SendAsync("Fetch.disable")).ConfigureAwait(false);
558            }
559        }
560    }
561}
562
Full Screen

Accelerate Your Automation Test Cycles With LambdaTest

Leverage LambdaTest’s cloud-based platform to execute your automation tests in parallel and trim down your test execution time significantly. Your first 100 automation testing minutes are on us.

Try LambdaTest

Trigger ForgetRequest code on LambdaTest Cloud Grid

Execute automation tests with ForgetRequest on a cloud-based Grid of 3000+ real browsers and operating systems for both web and mobile applications.

Test now for Free
LambdaTestX

We use cookies to give you the best experience. Cookies help to provide a more personalized experience and relevant advertising for you, and web analytics for us. Learn More in our Cookies policy, Privacy & Terms of service

Allow Cookie
Sarah

I hope you find the best code examples for your project.

If you want to accelerate automated browser testing, try LambdaTest. Your first 100 automation testing minutes are FREE.

Sarah Elson (Product & Growth Lead)