[{"data":1,"prerenderedAt":2526},["ShallowReactive",2],{"navLinks":3,"sidebar_docs_navigation_\u002Fdocs\u002Fshield-base":64,"navigation":138,"navLinks_footer":802,"\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fcrawlers_page":815,"\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fcrawlers_surround":1796,"\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fcrawlers":1799},{"id":4,"extension":5,"links":6,"meta":61,"stem":62,"__hash__":63},"navigationMenu\u002Fnavigation.json","json",[7,52,57],{"nested":8,"label":9,"icon":10,"to":11,"children":12},true,"Docs","i-lucide-book-open","\u002Fdocs\u002Fgetting-started",[13,19,26,32,39,45],{"label":14,"icon":15,"to":11,"description":16,"github":17,"badge":18},"Getting Started","i-lucide-rocket","An introduction to help you understand the core components.","https:\u002F\u002Fgithub.com\u002FSergo706\u002Fdocshub","Start Here",{"label":20,"icon":21,"to":22,"description":23,"github":24,"badge":25},"Auth H3 Client","i-lucide-key-round","\u002Fdocs\u002Fauth-h3client","Seamlessly enforce OAuth 2.0 authentication and session management integrated directly as the client of the IAM module.","https:\u002F\u002Fgithub.com\u002FSergo706\u002Fauth-h3client","Core",{"label":27,"icon":28,"to":29,"description":30,"github":31,"badge":25},"IAM","i-lucide-shield-check","\u002Fdocs\u002Fiam","Identity and Access Management featuring granular roles, permissions, and security policies.","https:\u002F\u002Fgithub.com\u002FSergo706\u002Fauth",{"label":33,"icon":34,"to":35,"description":36,"github":37,"badge":38},"Bot Detection","i-lucide-cpu","\u002Fdocs\u002Fbot-detection","Advanced behavioral analysis and request fingerprinting to stop malicious automated traffic.","https:\u002F\u002Fgithub.com\u002FSergo706\u002Fbot-detector","Security",{"label":40,"icon":41,"to":42,"description":43,"github":44,"badge":38},"Shield Base","i-lucide-database-zap","\u002Fdocs\u002Fshield-base","CLI and programmatic toolkit for compiling offline-ready IP intelligence databases from BGP, GeoIP, Tor, FireHOL, and other public threat feeds.","https:\u002F\u002Fgithub.com\u002FSergo706\u002Fshield-base-cli",{"label":46,"icon":47,"to":48,"description":49,"github":50,"badge":51},"Utils","i-lucide-wrench","\u002Fdocs\u002Futils","A standard library of highly optimized helpers for formatting, validation, and core logic.","https:\u002F\u002Fgithub.com\u002FSergo706\u002Futils","Library",{"nested":53,"label":54,"icon":55,"to":56},false,"Blog","i-lucide-pen-line","\u002Fblog",{"nested":53,"label":58,"icon":59,"to":60},"Website","lucide:app-window-mac","https:\u002F\u002Friavzon.com",{},"navigation","gkaQ0xRGxSLrLyM3kttLe0oBwkrR1EBjlepF8LSbwF8",[65],{"title":9,"path":66,"stem":67,"children":68,"page":53},"\u002Fdocs","docs",[69],{"title":40,"path":42,"stem":70,"children":71},"docs\u002Fshield-base\u002Findex",[72,73,76,80,122,126,130,134],{"title":40,"path":42,"stem":70},{"title":14,"path":74,"stem":75},"\u002Fdocs\u002Fshield-base\u002Fgetting-started","docs\u002Fshield-base\u002F00.getting-started",{"title":77,"path":78,"stem":79},"CLI Reference","\u002Fdocs\u002Fshield-base\u002Fcli","docs\u002Fshield-base\u002F01.cli",{"title":81,"path":82,"stem":83,"children":84},"Data Sources","\u002Fdocs\u002Fshield-base\u002Fdata-sources","docs\u002Fshield-base\u002F02.data-sources\u002Findex",[85,86,90,94,98,102,106,110,114,118],{"title":81,"path":82,"stem":83},{"title":87,"path":88,"stem":89},"BGP \u002F ASN","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fbgp","docs\u002Fshield-base\u002F02.data-sources\u002Fbgp",{"title":91,"path":92,"stem":93},"City Geolocation","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fcity","docs\u002Fshield-base\u002F02.data-sources\u002Fcity",{"title":95,"path":96,"stem":97},"Country Geolocation","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fcountry","docs\u002Fshield-base\u002F02.data-sources\u002Fcountry",{"title":99,"path":100,"stem":101},"Verified Crawlers","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fcrawlers","docs\u002Fshield-base\u002F02.data-sources\u002Fcrawlers",{"title":103,"path":104,"stem":105},"Disposable Emails","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Femail","docs\u002Fshield-base\u002F02.data-sources\u002Femail",{"title":107,"path":108,"stem":109},"FireHOL Threat Intelligence","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Ffirehol","docs\u002Fshield-base\u002F02.data-sources\u002Ffirehol",{"title":111,"path":112,"stem":113},"Proxy Detection","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fproxy","docs\u002Fshield-base\u002F02.data-sources\u002Fproxy",{"title":115,"path":116,"stem":117},"Tor Nodes","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Ftor","docs\u002Fshield-base\u002F02.data-sources\u002Ftor",{"title":119,"path":120,"stem":121},"Suspicious User-Agents","\u002Fdocs\u002Fshield-base\u002Fdata-sources\u002Fuseragent","docs\u002Fshield-base\u002F02.data-sources\u002Fuseragent",{"title":123,"path":124,"stem":125},"Programmatic Usage","\u002Fdocs\u002Fshield-base\u002Fusage","docs\u002Fshield-base\u002F03.usage",{"title":127,"path":128,"stem":129},"Custom Data Sources","\u002Fdocs\u002Fshield-base\u002Fcustom-data-sources","docs\u002Fshield-base\u002F04.custom-data-sources",{"title":131,"path":132,"stem":133},"TypeScript Types","\u002Fdocs\u002Fshield-base\u002Ftypes","docs\u002Fshield-base\u002F05.types",{"title":135,"path":136,"stem":137},"API Reference","\u002Fdocs\u002Fshield-base\u002Fapi","docs\u002Fshield-base\u002F06.api",[139],{"title":9,"path":66,"stem":67,"children":140,"page":53},[141,288,405,410,588,609],{"title":20,"path":22,"stem":142,"children":143},"docs\u002Fauth-h3client\u002Findex",[144,145,154,191,217,239,242,263,267],{"title":20,"path":22,"stem":142},{"title":14,"path":146,"stem":147,"children":148},"\u002Fdocs\u002Fauth-h3client\u002Fgetting-started","docs\u002Fauth-h3client\u002F00.getting-started\u002Findex",[149,150],{"title":14,"path":146,"stem":147},{"title":151,"path":152,"stem":153},"Nuxt Module","\u002Fdocs\u002Fauth-h3client\u002Fgetting-started\u002Fnuxt","docs\u002Fauth-h3client\u002F00.getting-started\u002F00.nuxt",{"title":155,"path":156,"stem":157,"children":158},"Essentials","\u002Fdocs\u002Fauth-h3client\u002Fessentials","docs\u002Fauth-h3client\u002F01.essentials\u002Findex",[159,160,164,168,172,176,180,183,187],{"title":155,"path":156,"stem":157},{"title":161,"path":162,"stem":163},"Session Management","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Fsession","docs\u002Fauth-h3client\u002F01.essentials\u002F00.session",{"title":165,"path":166,"stem":167},"Route Protection","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Froute-protection","docs\u002Fauth-h3client\u002F01.essentials\u002F01.route-protection",{"title":169,"path":170,"stem":171},"CSRF Protection","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Fcsrf","docs\u002Fauth-h3client\u002F01.essentials\u002F02.csrf",{"title":173,"path":174,"stem":175},"Auth Flows","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Fauth-flows","docs\u002Fauth-h3client\u002F01.essentials\u002F03.auth-flows",{"title":177,"path":178,"stem":179},"OAuth and OIDC","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Foauth","docs\u002Fauth-h3client\u002F01.essentials\u002F04.oauth",{"title":33,"path":181,"stem":182},"\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Fbot-detection","docs\u002Fauth-h3client\u002F01.essentials\u002F05.bot-detection",{"title":184,"path":185,"stem":186},"Cookies","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Fcookies","docs\u002Fauth-h3client\u002F01.essentials\u002F06.cookies",{"title":188,"path":189,"stem":190},"Logging","\u002Fdocs\u002Fauth-h3client\u002Fessentials\u002Flogging","docs\u002Fauth-h3client\u002F01.essentials\u002F07.logging",{"title":192,"path":193,"stem":194,"children":195},"MFA","\u002Fdocs\u002Fauth-h3client\u002Fmfa","docs\u002Fauth-h3client\u002F02.mfa\u002Findex",[196,197,201,205,209,213],{"title":192,"path":193,"stem":194},{"title":198,"path":199,"stem":200},"Built-in MFA","\u002Fdocs\u002Fauth-h3client\u002Fmfa\u002Fbuilt-in-flow","docs\u002Fauth-h3client\u002F02.mfa\u002F01.built-in-flow",{"title":202,"path":203,"stem":204},"Password Reset","\u002Fdocs\u002Fauth-h3client\u002Fmfa\u002Fpassword-reset","docs\u002Fauth-h3client\u002F02.mfa\u002F02.password-reset",{"title":206,"path":207,"stem":208},"Email Change","\u002Fdocs\u002Fauth-h3client\u002Fmfa\u002Femail-change","docs\u002Fauth-h3client\u002F02.mfa\u002F03.email-change",{"title":210,"path":211,"stem":212},"Custom MFA Flow","\u002Fdocs\u002Fauth-h3client\u002Fmfa\u002Fcustom-flow","docs\u002Fauth-h3client\u002F02.mfa\u002F04.custom-flow",{"title":214,"path":215,"stem":216},"Client-Side MFA","\u002Fdocs\u002Fauth-h3client\u002Fmfa\u002Fclient-side","docs\u002Fauth-h3client\u002F02.mfa\u002F05.client-side",{"title":218,"path":219,"stem":220,"children":221},"Client-side","\u002Fdocs\u002Fauth-h3client\u002Fclient","docs\u002Fauth-h3client\u002F03.client\u002Findex",[222,223,227,231,235],{"title":218,"path":219,"stem":220},{"title":224,"path":225,"stem":226},"useAuthData","\u002Fdocs\u002Fauth-h3client\u002Fclient\u002Fuse-auth-data","docs\u002Fauth-h3client\u002F03.client\u002F00.use-auth-data",{"title":228,"path":229,"stem":230},"useMagicLink","\u002Fdocs\u002Fauth-h3client\u002Fclient\u002Fuse-magic-link","docs\u002Fauth-h3client\u002F03.client\u002F01.use-magic-link",{"title":232,"path":233,"stem":234},"executeRequest","\u002Fdocs\u002Fauth-h3client\u002Fclient\u002Fexecute-request","docs\u002Fauth-h3client\u002F03.client\u002F02.execute-request",{"title":236,"path":237,"stem":238},"getCsrfToken","\u002Fdocs\u002Fauth-h3client\u002Fclient\u002Fget-csrf-token","docs\u002Fauth-h3client\u002F03.client\u002F03.get-csrf-token",{"title":38,"path":240,"stem":241},"\u002Fdocs\u002Fauth-h3client\u002Fsecurity","docs\u002Fauth-h3client\u002F04.security",{"title":243,"path":244,"stem":245,"children":246,"page":53},"Guides","\u002Fdocs\u002Fauth-h3client\u002Fguides","docs\u002Fauth-h3client\u002F05.guides",[247,251,255,259],{"title":248,"path":249,"stem":250},"H3 and Nitro Setup","\u002Fdocs\u002Fauth-h3client\u002Fguides\u002Fh3-nitro","docs\u002Fauth-h3client\u002F05.guides\u002F00.h3-nitro",{"title":252,"path":253,"stem":254},"HMAC Inter-service Auth","\u002Fdocs\u002Fauth-h3client\u002Fguides\u002Fhmac","docs\u002Fauth-h3client\u002F05.guides\u002Fhmac",{"title":256,"path":257,"stem":258},"Image Upload","\u002Fdocs\u002Fauth-h3client\u002Fguides\u002Fimage-upload","docs\u002Fauth-h3client\u002F05.guides\u002Fimage-upload",{"title":260,"path":261,"stem":262},"mTLS Configuration","\u002Fdocs\u002Fauth-h3client\u002Fguides\u002Fmtls","docs\u002Fauth-h3client\u002F05.guides\u002Fmtls",{"title":264,"path":265,"stem":266},"Configuration","\u002Fdocs\u002Fauth-h3client\u002Fconfiguration","docs\u002Fauth-h3client\u002F06.configuration",{"title":135,"path":268,"stem":269,"children":270},"\u002Fdocs\u002Fauth-h3client\u002Fapi","docs\u002Fauth-h3client\u002F07.api\u002Findex",[271,272,276,280,284],{"title":135,"path":268,"stem":269},{"title":273,"path":274,"stem":275},"Routes Reference","\u002Fdocs\u002Fauth-h3client\u002Fapi\u002Fcontrollers","docs\u002Fauth-h3client\u002F07.api\u002F00.controllers",{"title":277,"path":278,"stem":279},"Middleware Reference","\u002Fdocs\u002Fauth-h3client\u002Fapi\u002Fmiddleware","docs\u002Fauth-h3client\u002F07.api\u002F01.middleware",{"title":281,"path":282,"stem":283},"Client-side Reference","\u002Fdocs\u002Fauth-h3client\u002Fapi\u002Fcomposables","docs\u002Fauth-h3client\u002F07.api\u002F02.composables",{"title":285,"path":286,"stem":287},"Utilities","\u002Fdocs\u002Fauth-h3client\u002Fapi\u002Futilities","docs\u002Fauth-h3client\u002F07.api\u002F03.utilities",{"title":289,"path":35,"stem":290,"children":291},"Bot Detector","docs\u002Fbot-detection\u002Findex",[292,293,296,300,303,322,396,399,402],{"title":289,"path":35,"stem":290},{"title":14,"path":294,"stem":295},"\u002Fdocs\u002Fbot-detection\u002Fgetting-started","docs\u002Fbot-detection\u002F00.getting-started",{"title":297,"path":298,"stem":299},"CLI","\u002Fdocs\u002Fbot-detection\u002Fcli","docs\u002Fbot-detection\u002F01.cli",{"title":81,"path":301,"stem":302},"\u002Fdocs\u002Fbot-detection\u002Fdata-sources","docs\u002Fbot-detection\u002F02.data-sources",{"title":243,"path":304,"stem":305,"children":306,"page":53},"\u002Fdocs\u002Fbot-detection\u002Fguides","docs\u002Fbot-detection\u002F03.guides",[307,311,315,318],{"title":308,"path":309,"stem":310},"Custom Checkers","\u002Fdocs\u002Fbot-detection\u002Fguides\u002Fcustom","docs\u002Fbot-detection\u002F03.guides\u002FCUSTOM",{"title":312,"path":313,"stem":314},"Scheduling Database Generation","\u002Fdocs\u002Fbot-detection\u002Fguides\u002Fgenerate","docs\u002Fbot-detection\u002F03.guides\u002FGENERATE",{"title":188,"path":316,"stem":317},"\u002Fdocs\u002Fbot-detection\u002Fguides\u002Flogging","docs\u002Fbot-detection\u002F03.guides\u002FLOGGING",{"title":319,"path":320,"stem":321},"Score Modes and Reputation Healing","\u002Fdocs\u002Fbot-detection\u002Fguides\u002Fscore","docs\u002Fbot-detection\u002F03.guides\u002FSCORE",{"title":323,"path":324,"stem":325,"children":326},"Checkers","\u002Fdocs\u002Fbot-detection\u002Fcheckers","docs\u002Fbot-detection\u002F04.checkers\u002Findex",[327,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392],{"title":323,"path":324,"stem":325},{"title":329,"path":330,"stem":331},"IP Validation","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fip-validation","docs\u002Fbot-detection\u002F04.checkers\u002F01.ip-validation",{"title":333,"path":334,"stem":335},"Good \u002F Bad Bot Verification","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fgood-bots","docs\u002Fbot-detection\u002F04.checkers\u002F02.good-bots",{"title":337,"path":338,"stem":339},"Browser & Device Fingerprint","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fbrowser-device","docs\u002Fbot-detection\u002F04.checkers\u002F03.browser-device",{"title":341,"path":342,"stem":343},"Locale Map","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Flocale-map","docs\u002Fbot-detection\u002F04.checkers\u002F04.locale-map",{"title":345,"path":346,"stem":347},"Known Threats","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fknown-threats","docs\u002Fbot-detection\u002F04.checkers\u002F05.known-threats",{"title":349,"path":350,"stem":351},"ASN Classification","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fasn-classification","docs\u002Fbot-detection\u002F04.checkers\u002F06.asn-classification",{"title":353,"path":354,"stem":355},"Tor Analysis","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Ftor-analysis","docs\u002Fbot-detection\u002F04.checkers\u002F07.tor-analysis",{"title":357,"path":358,"stem":359},"Timezone Consistency","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Ftimezone-consistency","docs\u002Fbot-detection\u002F04.checkers\u002F08.timezone-consistency",{"title":361,"path":362,"stem":363},"Honeypot","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fhoneypot","docs\u002Fbot-detection\u002F04.checkers\u002F09.honeypot",{"title":365,"path":366,"stem":367},"Known Bad IPs","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fknown-bad-ips","docs\u002Fbot-detection\u002F04.checkers\u002F10.known-bad-ips",{"title":369,"path":370,"stem":371},"Behavior Rate","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fbehavior-rate","docs\u002Fbot-detection\u002F04.checkers\u002F11.behavior-rate",{"title":373,"path":374,"stem":375},"Proxy \u002F ISP \u002F Cookie","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fproxy-isp-cookies","docs\u002Fbot-detection\u002F04.checkers\u002F12.proxy-isp-cookies",{"title":377,"path":378,"stem":379},"Session Coherence","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fsession-coherence","docs\u002Fbot-detection\u002F04.checkers\u002F13.session-coherence",{"title":381,"path":382,"stem":383},"Velocity Fingerprint","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fvelocity-fingerprint","docs\u002Fbot-detection\u002F04.checkers\u002F14.velocity-fingerprint",{"title":385,"path":386,"stem":387},"UA & Header Analysis","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fua-header","docs\u002Fbot-detection\u002F04.checkers\u002F15.ua-header",{"title":389,"path":390,"stem":391},"Geolocation","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fgeolocation","docs\u002Fbot-detection\u002F04.checkers\u002F16.geolocation",{"title":393,"path":394,"stem":395},"Known Bad User-Agents","\u002Fdocs\u002Fbot-detection\u002Fcheckers\u002Fknown-bad-ua","docs\u002Fbot-detection\u002F04.checkers\u002F17.known-bad-ua",{"title":38,"path":397,"stem":398},"\u002Fdocs\u002Fbot-detection\u002Fsecurity","docs\u002Fbot-detection\u002F04.security",{"title":135,"path":400,"stem":401},"\u002Fdocs\u002Fbot-detection\u002Fapi","docs\u002Fbot-detection\u002F05.api",{"title":264,"path":403,"stem":404},"\u002Fdocs\u002Fbot-detection\u002Fconfiguration","docs\u002Fbot-detection\u002F06.configuration",{"title":406,"path":11,"stem":407,"children":408},"Introduction","docs\u002Fgetting-started\u002Findex",[409],{"title":406,"path":11,"stem":407},{"title":27,"path":29,"stem":411,"children":412},"docs\u002Fiam\u002Findex",[413,414,417,552,555,571,574],{"title":27,"path":29,"stem":411},{"title":14,"path":415,"stem":416},"\u002Fdocs\u002Fiam\u002Fgetting-started","docs\u002Fiam\u002F00.getting-started",{"title":155,"path":418,"stem":419,"children":420},"\u002Fdocs\u002Fiam\u002Fessentials","docs\u002Fiam\u002F01.essentials\u002Findex",[421,422,426,430,434,438,442,446,450,454,458,462,465,469,473,477,481,484,488,492,495,499,502],{"title":155,"path":418,"stem":419},{"title":423,"path":424,"stem":425},"Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Ftokens","docs\u002Fiam\u002F01.essentials\u002F00.tokens",{"title":427,"path":428,"stem":429},"Access Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Faccess-tokens","docs\u002Fiam\u002F01.essentials\u002F01.access-tokens",{"title":431,"path":432,"stem":433},"Refresh Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Frefresh-tokens","docs\u002Fiam\u002F01.essentials\u002F02.refresh-tokens",{"title":435,"path":436,"stem":437},"Anomaly Detection","\u002Fdocs\u002Fiam\u002Fessentials\u002Fanomalies","docs\u002Fiam\u002F01.essentials\u002F03.anomalies",{"title":439,"path":440,"stem":441},"Signup","\u002Fdocs\u002Fiam\u002Fessentials\u002Fsignup","docs\u002Fiam\u002F01.essentials\u002F04.signup",{"title":443,"path":444,"stem":445},"Login","\u002Fdocs\u002Fiam\u002Fessentials\u002Flogin","docs\u002Fiam\u002F01.essentials\u002F05.login",{"title":447,"path":448,"stem":449},"Logout","\u002Fdocs\u002Fiam\u002Fessentials\u002Flogout","docs\u002Fiam\u002F01.essentials\u002F06.logout",{"title":451,"path":452,"stem":453},"OAuth","\u002Fdocs\u002Fiam\u002Fessentials\u002Foauth","docs\u002Fiam\u002F01.essentials\u002F07.oauth",{"title":455,"path":456,"stem":457},"Magic Links","\u002Fdocs\u002Fiam\u002Fessentials\u002Fmagic-links","docs\u002Fiam\u002F01.essentials\u002F08.magic-links",{"title":459,"path":460,"stem":461},"Emails","\u002Fdocs\u002Fiam\u002Fessentials\u002Femails","docs\u002Fiam\u002F01.essentials\u002F09.emails",{"title":192,"path":463,"stem":464},"\u002Fdocs\u002Fiam\u002Fessentials\u002Fmfa","docs\u002Fiam\u002F01.essentials\u002F10.mfa",{"title":466,"path":467,"stem":468},"Fingerprinting","\u002Fdocs\u002Fiam\u002Fessentials\u002Ffingerprinting","docs\u002Fiam\u002F01.essentials\u002F11.fingerprinting",{"title":470,"path":471,"stem":472},"Backend for Frontend","\u002Fdocs\u002Fiam\u002Fessentials\u002Fbff","docs\u002Fiam\u002F01.essentials\u002F12.bff",{"title":474,"path":475,"stem":476},"HMAC Authentication","\u002Fdocs\u002Fiam\u002Fessentials\u002Fhmac","docs\u002Fiam\u002F01.essentials\u002F13.hmac",{"title":478,"path":479,"stem":480},"XSS Protection","\u002Fdocs\u002Fiam\u002Fessentials\u002Fxss","docs\u002Fiam\u002F01.essentials\u002F14.xss",{"title":188,"path":482,"stem":483},"\u002Fdocs\u002Fiam\u002Fessentials\u002Flogging","docs\u002Fiam\u002F01.essentials\u002F15.logging",{"title":485,"path":486,"stem":487},"Rate Limiting","\u002Fdocs\u002Fiam\u002Fessentials\u002Frate-limiting","docs\u002Fiam\u002F01.essentials\u002F16.rate-limiting",{"title":489,"path":490,"stem":491},"Database","\u002Fdocs\u002Fiam\u002Fessentials\u002Fdatabase","docs\u002Fiam\u002F01.essentials\u002F17.database",{"title":184,"path":493,"stem":494},"\u002Fdocs\u002Fiam\u002Fessentials\u002Fcookies","docs\u002Fiam\u002F01.essentials\u002F18.cookies",{"title":496,"path":497,"stem":498},"Service Startup","\u002Fdocs\u002Fiam\u002Fessentials\u002Fservice","docs\u002Fiam\u002F01.essentials\u002F19.service",{"title":202,"path":500,"stem":501},"\u002Fdocs\u002Fiam\u002Fessentials\u002Fpassword-reset","docs\u002Fiam\u002F01.essentials\u002F20.password-reset",{"title":503,"path":504,"stem":505,"children":506},"API Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi","docs\u002Fiam\u002F01.essentials\u002F21.api\u002Findex",[507,508,512,516,546,549],{"title":503,"path":504,"stem":505},{"title":509,"path":510,"stem":511},"Creating Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fcreation","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F00.creation",{"title":513,"path":514,"stem":515},"Verifying Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fverification","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F01.verification",{"title":517,"path":518,"stem":519,"children":520},"Manage Tokens","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002Findex",[521,522,526,530,534,538,542],{"title":517,"path":518,"stem":519},{"title":523,"path":524,"stem":525},"Privileges","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement\u002Fprivilege","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002F00.privilege",{"title":527,"path":528,"stem":529},"Revocation","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement\u002Frevocation","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002F01.revocation",{"title":531,"path":532,"stem":533},"Rotation","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement\u002Frotation","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002F02.rotation",{"title":535,"path":536,"stem":537},"IP Restriction","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement\u002Fip-updates","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002F03.ip-updates",{"title":539,"path":540,"stem":541},"Metadata","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement\u002Fmetadata","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002F04.metadata",{"title":543,"path":544,"stem":545},"Token Listing","\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fmanagement\u002Flist","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F02.management\u002F05.list",{"title":485,"path":547,"stem":548},"\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Frate-limiting","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F03.rate-limiting",{"title":38,"path":550,"stem":551},"\u002Fdocs\u002Fiam\u002Fessentials\u002Fapi\u002Fsecurity","docs\u002Fiam\u002F01.essentials\u002F21.api\u002F04.security",{"title":38,"path":553,"stem":554},"\u002Fdocs\u002Fiam\u002Fsecurity","docs\u002Fiam\u002F02.security",{"title":243,"path":556,"stem":557,"children":558,"page":53},"\u002Fdocs\u002Fiam\u002Fguides","docs\u002Fiam\u002F03.guides",[559,563,567],{"title":560,"path":561,"stem":562},"Deployment","\u002Fdocs\u002Fiam\u002Fguides\u002Fdeployment","docs\u002Fiam\u002F03.guides\u002Fdeployment",{"title":564,"path":565,"stem":566},"Operation Scripts","\u002Fdocs\u002Fiam\u002Fguides\u002Foperation-scripts","docs\u002Fiam\u002F03.guides\u002Foperation-scripts",{"title":568,"path":569,"stem":570},"Role-Based Access Control","\u002Fdocs\u002Fiam\u002Fguides\u002Frbac","docs\u002Fiam\u002F03.guides\u002Frbac",{"title":264,"path":572,"stem":573},"\u002Fdocs\u002Fiam\u002Fconfiguration","docs\u002Fiam\u002F04.configuration",{"title":575,"path":576,"stem":577,"children":578,"page":53},"Api","\u002Fdocs\u002Fiam\u002Fapi","docs\u002Fiam\u002F05.API",[579,582,585],{"title":135,"path":580,"stem":581},"\u002Fdocs\u002Fiam\u002Fapi\u002Fapi","docs\u002Fiam\u002F05.API\u002F00.api",{"title":277,"path":583,"stem":584},"\u002Fdocs\u002Fiam\u002Fapi\u002Fmiddlewares","docs\u002Fiam\u002F05.API\u002F02.middlewares",{"title":273,"path":586,"stem":587},"\u002Fdocs\u002Fiam\u002Fapi\u002Froutes","docs\u002Fiam\u002F05.API\u002F03.routes",{"title":40,"path":42,"stem":70,"children":589},[590,591,592,593,605,606,607,608],{"title":40,"path":42,"stem":70},{"title":14,"path":74,"stem":75},{"title":77,"path":78,"stem":79},{"title":81,"path":82,"stem":83,"children":594},[595,596,597,598,599,600,601,602,603,604],{"title":81,"path":82,"stem":83},{"title":87,"path":88,"stem":89},{"title":91,"path":92,"stem":93},{"title":95,"path":96,"stem":97},{"title":99,"path":100,"stem":101},{"title":103,"path":104,"stem":105},{"title":107,"path":108,"stem":109},{"title":111,"path":112,"stem":113},{"title":115,"path":116,"stem":117},{"title":119,"path":120,"stem":121},{"title":123,"path":124,"stem":125},{"title":127,"path":128,"stem":129},{"title":131,"path":132,"stem":133},{"title":135,"path":136,"stem":137},{"title":285,"path":48,"stem":610,"children":611},"docs\u002Futils\u002Findex",[612,613,630,663,760],{"title":285,"path":48,"stem":610},{"title":614,"path":615,"stem":616,"children":617,"page":53},"Eslint","\u002Fdocs\u002Futils\u002Feslint","docs\u002Futils\u002Feslint",[618,622,626],{"title":619,"path":620,"stem":621},"React Config","\u002Fdocs\u002Futils\u002Feslint\u002Freact","docs\u002Futils\u002Feslint\u002Freact",{"title":623,"path":624,"stem":625},"TypeScript Config","\u002Fdocs\u002Futils\u002Feslint\u002Ftypescript","docs\u002Futils\u002Feslint\u002Ftypescript",{"title":627,"path":628,"stem":629},"Vue Config","\u002Fdocs\u002Futils\u002Feslint\u002Fvue","docs\u002Futils\u002Feslint\u002Fvue",{"title":631,"path":632,"stem":633,"children":634,"page":53},"Server","\u002Fdocs\u002Futils\u002Fserver","docs\u002Futils\u002Fserver",[635,639,643,647,651,655,659],{"title":636,"path":637,"stem":638},"Encryption","\u002Fdocs\u002Futils\u002Fserver\u002Fencryption","docs\u002Futils\u002Fserver\u002Fencryption",{"title":640,"path":641,"stem":642},"Path Resolver","\u002Fdocs\u002Futils\u002Fserver\u002Fpathresolver","docs\u002Futils\u002Fserver\u002FpathResolver",{"title":644,"path":645,"stem":646},"File Replacements","\u002Fdocs\u002Futils\u002Fserver\u002Freplace","docs\u002Futils\u002Fserver\u002Freplace",{"title":648,"path":649,"stem":650},"run","\u002Fdocs\u002Futils\u002Fserver\u002Frun","docs\u002Futils\u002Fserver\u002Frun",{"title":652,"path":653,"stem":654},"scheduleTask","\u002Fdocs\u002Futils\u002Fserver\u002Fscheduletask","docs\u002Futils\u002Fserver\u002FscheduleTask",{"title":656,"path":657,"stem":658},"spawnRun","\u002Fdocs\u002Futils\u002Fserver\u002Fspawnrun","docs\u002Futils\u002Fserver\u002FspawnRun",{"title":660,"path":661,"stem":662},"uploadCsv","\u002Fdocs\u002Futils\u002Fserver\u002Fuploadcsv","docs\u002Futils\u002Fserver\u002FuploadCsv",{"title":664,"path":665,"stem":666,"children":667,"page":53},"Shared","\u002Fdocs\u002Futils\u002Fshared","docs\u002Futils\u002Fshared",[668,672,676,680,684,688,692,696,700,704,708,712,716,720,724,728,732,736,740,744,748,752,756],{"title":669,"path":670,"stem":671},"BatchQueue","\u002Fdocs\u002Futils\u002Fshared\u002Fbatchqueue","docs\u002Futils\u002Fshared\u002FbatchQueue",{"title":673,"path":674,"stem":675},"capitalize","\u002Fdocs\u002Futils\u002Fshared\u002Fcapitalize","docs\u002Futils\u002Fshared\u002Fcapitalize",{"title":677,"path":678,"stem":679},"chunkProcess","\u002Fdocs\u002Futils\u002Fshared\u002Fchunkprocess","docs\u002Futils\u002Fshared\u002FchunkProcess",{"title":681,"path":682,"stem":683},"cleanObject","\u002Fdocs\u002Futils\u002Fshared\u002Fcleanobject","docs\u002Futils\u002Fshared\u002FcleanObject",{"title":685,"path":686,"stem":687},"createConfigManager","\u002Fdocs\u002Futils\u002Fshared\u002Fconfigurationdefiner","docs\u002Futils\u002Fshared\u002FconfigurationDefiner",{"title":689,"path":690,"stem":691},"debounce","\u002Fdocs\u002Futils\u002Fshared\u002Fdebounce","docs\u002Futils\u002Fshared\u002Fdebounce",{"title":693,"path":694,"stem":695},"ensureArray","\u002Fdocs\u002Futils\u002Fshared\u002Fensurearray","docs\u002Futils\u002Fshared\u002FensureArray",{"title":697,"path":698,"stem":699},"fetchWithRetry","\u002Fdocs\u002Futils\u002Fshared\u002Ffetchwithretry","docs\u002Futils\u002Fshared\u002FfetchWithRetry",{"title":701,"path":702,"stem":703},"filterEmptyValues","\u002Fdocs\u002Futils\u002Fshared\u002Ffilteremptyvalues","docs\u002Futils\u002Fshared\u002FfilterEmptyValues",{"title":705,"path":706,"stem":707},"findStringsInObject","\u002Fdocs\u002Futils\u002Fshared\u002Ffindobjectvalues","docs\u002Futils\u002Fshared\u002FfindObjectValues",{"title":709,"path":710,"stem":711},"fisherYatesShuffle","\u002Fdocs\u002Futils\u002Fshared\u002Ffisheryatesshuffle","docs\u002Futils\u002Fshared\u002FfisherYatesShuffle",{"title":713,"path":714,"stem":715},"getRandomImage","\u002Fdocs\u002Futils\u002Fshared\u002Fgetrandomimage","docs\u002Futils\u002Fshared\u002FgetRandomImage",{"title":717,"path":718,"stem":719},"isObjectHasValues","\u002Fdocs\u002Futils\u002Fshared\u002Fisobjecthasvalues","docs\u002Futils\u002Fshared\u002FisObjectHasValues",{"title":721,"path":722,"stem":723},"isAsyncOrPromise","\u002Fdocs\u002Futils\u002Fshared\u002Fispromise","docs\u002Futils\u002Fshared\u002FisPromise",{"title":725,"path":726,"stem":727},"MiniCache","\u002Fdocs\u002Futils\u002Fshared\u002Fminicache","docs\u002Futils\u002Fshared\u002FminiCache",{"title":729,"path":730,"stem":731},"parseCookies","\u002Fdocs\u002Futils\u002Fshared\u002Fparserawcookies","docs\u002Futils\u002Fshared\u002FparseRawCookies",{"title":733,"path":734,"stem":735},"safeAction","\u002Fdocs\u002Futils\u002Fshared\u002Fpromiselocker","docs\u002Futils\u002Fshared\u002FpromiseLocker",{"title":737,"path":738,"stem":739},"Random","\u002Fdocs\u002Futils\u002Fshared\u002Frandom","docs\u002Futils\u002Fshared\u002Frandom",{"title":741,"path":742,"stem":743},"range","\u002Fdocs\u002Futils\u002Fshared\u002Frange","docs\u002Futils\u002Fshared\u002Frange",{"title":745,"path":746,"stem":747},"rateLimiters","\u002Fdocs\u002Futils\u002Fshared\u002Fratelimiters","docs\u002Futils\u002Fshared\u002FrateLimiters",{"title":749,"path":750,"stem":751},"safeObjectMerge","\u002Fdocs\u002Futils\u002Fshared\u002Fsafemerge","docs\u002Futils\u002Fshared\u002FsafeMerge",{"title":753,"path":754,"stem":755},"textTruncation","\u002Fdocs\u002Futils\u002Fshared\u002Ftexttruncation","docs\u002Futils\u002Fshared\u002FtextTruncation",{"title":757,"path":758,"stem":759},"validateZodSchema","\u002Fdocs\u002Futils\u002Fshared\u002Fvalidatezodschema","docs\u002Futils\u002Fshared\u002FvalidateZodSchema",{"title":761,"path":762,"stem":763,"children":764},"Utility Types","\u002Fdocs\u002Futils\u002Ftypes","docs\u002Futils\u002Ftypes\u002Findex",[765,766,770,774,778,782,786,790,794,798],{"title":761,"path":762,"stem":763},{"title":767,"path":768,"stem":769},"Brand","\u002Fdocs\u002Futils\u002Ftypes\u002Fbrand","docs\u002Futils\u002Ftypes\u002FBrand",{"title":771,"path":772,"stem":773},"DeepPartial","\u002Fdocs\u002Futils\u002Ftypes\u002Fdeeppartial","docs\u002Futils\u002Ftypes\u002FDeepPartial",{"title":775,"path":776,"stem":777},"Merge","\u002Fdocs\u002Futils\u002Ftypes\u002Fmerge","docs\u002Futils\u002Ftypes\u002FMerge",{"title":779,"path":780,"stem":781},"NonNullable","\u002Fdocs\u002Futils\u002Ftypes\u002Fnonnullable","docs\u002Futils\u002Ftypes\u002FNonNullable",{"title":783,"path":784,"stem":785},"Prettify","\u002Fdocs\u002Futils\u002Ftypes\u002Fprettify","docs\u002Futils\u002Ftypes\u002FPrettify",{"title":787,"path":788,"stem":789},"PromiseType","\u002Fdocs\u002Futils\u002Ftypes\u002Fpromisetype","docs\u002Futils\u002Ftypes\u002FPromiseType",{"title":791,"path":792,"stem":793},"RequireKeys","\u002Fdocs\u002Futils\u002Ftypes\u002Frequirekeys","docs\u002Futils\u002Ftypes\u002FRequireKeys",{"title":795,"path":796,"stem":797},"StandardResponse","\u002Fdocs\u002Futils\u002Ftypes\u002Fstandardresponse","docs\u002Futils\u002Ftypes\u002FStandardResponse",{"title":799,"path":800,"stem":801},"ValueOf","\u002Fdocs\u002Futils\u002Ftypes\u002Fvalueof","docs\u002Futils\u002Ftypes\u002FValueOf",{"id":4,"extension":5,"links":803,"meta":814,"stem":62,"__hash__":63},[804,812,813],{"nested":8,"label":9,"icon":10,"to":11,"children":805},[806,807,808,809,810,811],{"label":14,"icon":15,"to":11,"description":16,"github":17,"badge":18},{"label":20,"icon":21,"to":22,"description":23,"github":24,"badge":25},{"label":27,"icon":28,"to":29,"description":30,"github":31,"badge":25},{"label":33,"icon":34,"to":35,"description":36,"github":37,"badge":38},{"label":40,"icon":41,"to":42,"description":43,"github":44,"badge":38},{"label":46,"icon":47,"to":48,"description":49,"github":50,"badge":51},{"nested":53,"label":54,"icon":55,"to":56},{"nested":53,"label":58,"icon":59,"to":60},{},{"id":816,"title":99,"body":817,"description":1788,"extension":1789,"icon":1790,"meta":1791,"module":1792,"navigation":8,"path":100,"rawbody":1793,"seo":1794,"stem":101,"__hash__":1795},"docs\u002Fdocs\u002Fshield-base\u002F02.data-sources\u002Fcrawlers.md",{"type":818,"value":819,"toc":1782},"minimark",[820,824,832,842,851,854,859,1086,1088,1092,1453,1462,1464,1468,1632,1634,1638,1664,1762,1778],[821,822,823],"p",{},"The Crawlers source fetches IP ranges for major search engine crawlers and legitimate automated agents from their official geofeeds and documentation pages. Data is extracted from JSON, CSV, and HTML sources, then merged and compiled into a single MMDB database.",[821,825,826,827,831],{},"The fetcher uses a tiered fetch with a fallback mechanism that invokes ",[828,829,830],"code",{},"curl"," to bypass anti-scraping measures on social media geofeeds when it detects a regular fetch being blocked.",[821,833,834,838,839],{},[835,836,837],"strong",{},"Output file:"," ",[828,840,841],{},"goodBots.mmdb",[843,844,845],"warning",{},[821,846,847,848,850],{},"Make sure ",[828,849,830],{}," is installed on your system when using this data source. The fallback mechanism requires it to fetch certain provider pages.",[852,853],"hr",{},[855,856,858],"h2",{"id":857},"built-in-providers","Built-in Providers",[860,861,862,878],"table",{},[863,864,865],"thead",{},[866,867,868,872,875],"tr",{},[869,870,871],"th",{},"Provider",[869,873,874],{},"Type",[869,876,877],{},"Source",[879,880,881,920,934,953,967,981,1001,1015,1029,1044,1058,1072],"tbody",{},[866,882,883,887,890],{},[884,885,886],"td",{},"Google",[884,888,889],{},"JSON",[884,891,892,899,900,899,905,899,910,899,915],{},[893,894,898],"a",{"href":895,"rel":896},"https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fgooglebot.json",[897],"nofollow","googlebot.json",", ",[893,901,904],{"href":902,"rel":903},"https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fspecial-crawlers.json",[897],"special-crawlers.json",[893,906,909],{"href":907,"rel":908},"https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fuser-triggered-fetchers.json",[897],"user-triggered-fetchers.json",[893,911,914],{"href":912,"rel":913},"https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fuser-triggered-fetchers-google.json",[897],"user-triggered-fetchers-google.json",[893,916,919],{"href":917,"rel":918},"https:\u002F\u002Fwww.gstatic.com\u002Fipranges\u002Fgoog.json",[897],"goog.json",[866,921,922,925,927],{},[884,923,924],{},"Bing",[884,926,889],{},[884,928,929],{},[893,930,933],{"href":931,"rel":932},"https:\u002F\u002Fwww.bing.com\u002Ftoolbox\u002Fbingbot.json",[897],"bingbot.json",[866,935,936,939,941],{},[884,937,938],{},"OpenAI",[884,940,889],{},[884,942,943,899,948],{},[893,944,947],{"href":945,"rel":946},"https:\u002F\u002Fopenai.com\u002Fgptbot.json",[897],"gptbot.json",[893,949,952],{"href":950,"rel":951},"https:\u002F\u002Fopenai.com\u002Fsearchbot.json",[897],"searchbot.json",[866,954,955,958,960],{},[884,956,957],{},"Apple",[884,959,889],{},[884,961,962],{},[893,963,966],{"href":964,"rel":965},"https:\u002F\u002Fsearch.developer.apple.com\u002Fapplebot.json",[897],"applebot.json",[866,968,969,972,974],{},[884,970,971],{},"Ahrefs",[884,973,889],{},[884,975,976],{},[893,977,980],{"href":978,"rel":979},"https:\u002F\u002Fapi.ahrefs.com\u002Fv3\u002Fpublic\u002Fcrawler-ip-ranges",[897],"crawler-ip-ranges",[866,982,983,986,989],{},[884,984,985],{},"DuckDuckGo",[884,987,988],{},"HTML",[884,990,991,899,996],{},[893,992,995],{"href":993,"rel":994},"https:\u002F\u002Fduckduckgo.com\u002Fduckduckgo-help-pages\u002Fresults\u002Fduckassistbot",[897],"duckassistbot",[893,997,1000],{"href":998,"rel":999},"https:\u002F\u002Fduckduckgo.com\u002Fduckduckgo-help-pages\u002Fresults\u002Fduckduckbot",[897],"duckduckbot",[866,1002,1003,1006,1008],{},[884,1004,1005],{},"Common Crawl",[884,1007,988],{},[884,1009,1010],{},[893,1011,1014],{"href":1012,"rel":1013},"https:\u002F\u002Fcommoncrawl.org\u002Ffaq",[897],"commoncrawl.org\u002Ffaq",[866,1016,1017,1020,1022],{},[884,1018,1019],{},"X \u002F Twitter",[884,1021,988],{},[884,1023,1024],{},[893,1025,1028],{"href":1026,"rel":1027},"https:\u002F\u002Fdeveloper.x.com\u002Fen\u002Fdocs\u002Fx-for-websites\u002Fcards\u002Fguides\u002Ftroubleshooting-cards",[897],"troubleshooting-cards",[866,1030,1031,1034,1037],{},[884,1032,1033],{},"Facebook",[884,1035,1036],{},"CSV",[884,1038,1039],{},[893,1040,1043],{"href":1041,"rel":1042},"https:\u002F\u002Fwww.facebook.com\u002Fpeering\u002Fgeofeed",[897],"facebook.com\u002Fpeering\u002Fgeofeed",[866,1045,1046,1049,1051],{},[884,1047,1048],{},"Pinterest",[884,1050,988],{},[884,1052,1053],{},[893,1054,1057],{"href":1055,"rel":1056},"https:\u002F\u002Fhelp.pinterest.com\u002Fen\u002Fbusiness\u002Farticle\u002Fpinterestbot",[897],"pinterestbot",[866,1059,1060,1063,1065],{},[884,1061,1062],{},"Telegram",[884,1064,988],{},[884,1066,1067],{},[893,1068,1071],{"href":1069,"rel":1070},"https:\u002F\u002Fcore.telegram.org\u002Fbots\u002Fwebhooks",[897],"bots\u002Fwebhooks",[866,1073,1074,1077,1079],{},[884,1075,1076],{},"Semrush",[884,1078,988],{},[884,1080,1081],{},[893,1082,1085],{"href":1083,"rel":1084},"https:\u002F\u002Fwww.semrush.com\u002Fkb\u002F1149-issues-with-crawling-a-domain",[897],"semrush.com\u002Fkb\u002F1149",[852,1087],{},[855,1089,1091],{"id":1090},"usage","Usage",[1093,1094,1095,1124,1140,1155,1170],"code-group",{},[1096,1097,1103],"pre",{"className":1098,"code":1099,"filename":1100,"language":1101,"meta":1102,"style":1102},"language-bash shiki shiki-themes light-plus light-plus dracula","pnpm dlx @riavzon\u002Fshield-base --seo\n","pnpm","bash","",[828,1104,1105],{"__ignoreMap":1102},[1106,1107,1110,1113,1117,1120],"span",{"class":1108,"line":1109},"line",1,[1106,1111,1100],{"class":1112},"sHOzp",[1106,1114,1116],{"class":1115},"sFB1V"," dlx",[1106,1118,1119],{"class":1115}," @riavzon\u002Fshield-base",[1106,1121,1123],{"class":1122},"sjR7W"," --seo\n",[1096,1125,1128],{"className":1098,"code":1126,"filename":1127,"language":1101,"meta":1102,"style":1102},"yarn dlx @riavzon\u002Fshield-base --seo\n","yarn",[828,1129,1130],{"__ignoreMap":1102},[1106,1131,1132,1134,1136,1138],{"class":1108,"line":1109},[1106,1133,1127],{"class":1112},[1106,1135,1116],{"class":1115},[1106,1137,1119],{"class":1115},[1106,1139,1123],{"class":1122},[1096,1141,1144],{"className":1098,"code":1142,"filename":1143,"language":1101,"meta":1102,"style":1102},"npx @riavzon\u002Fshield-base --seo\n","npm",[828,1145,1146],{"__ignoreMap":1102},[1106,1147,1148,1151,1153],{"class":1108,"line":1109},[1106,1149,1150],{"class":1112},"npx",[1106,1152,1119],{"class":1115},[1106,1154,1123],{"class":1122},[1096,1156,1159],{"className":1098,"code":1157,"filename":1158,"language":1101,"meta":1102,"style":1102},"bunx @riavzon\u002Fshield-base --seo\n","bun",[828,1160,1161],{"__ignoreMap":1102},[1106,1162,1163,1166,1168],{"class":1108,"line":1109},[1106,1164,1165],{"class":1112},"bunx",[1106,1167,1119],{"class":1115},[1106,1169,1123],{"class":1122},[1096,1171,1176],{"className":1172,"code":1173,"filename":1174,"language":1175,"meta":1102,"style":1102},"language-ts shiki shiki-themes light-plus light-plus dracula","import { getCrawlersIps } from '@riavzon\u002Fshield-base';\n\n\u002F\u002F Compile with built-in providers only\nawait getCrawlersIps('.\u002Fout', 'mmdbctl');\n\n\u002F\u002F Or merge custom providers with built-in ones\nimport type { ProvidersLists } from '@riavzon\u002Fshield-base';\n\nconst customProviders: ProvidersLists[] = [\n  {\n    name: 'cloudflare',\n    type: 'JSON',\n    urls: [\n      'https:\u002F\u002Fwww.cloudflare.com\u002Fips-v4',\n      'https:\u002F\u002Fwww.cloudflare.com\u002Fips-v6',\n    ],\n  },\n];\n\nawait getCrawlersIps('.\u002Fout', 'mmdbctl', customProviders);\n","Programmatic","ts",[828,1177,1178,1211,1217,1224,1255,1260,1266,1291,1296,1324,1330,1349,1365,1375,1388,1400,1406,1412,1418,1423],{"__ignoreMap":1102},[1106,1179,1180,1184,1188,1192,1195,1198,1202,1205,1208],{"class":1108,"line":1109},[1106,1181,1183],{"class":1182},"sZ328","import",[1106,1185,1187],{"class":1186},"sDd4n"," { ",[1106,1189,1191],{"class":1190},"sjsA6","getCrawlersIps",[1106,1193,1194],{"class":1186}," } ",[1106,1196,1197],{"class":1182},"from",[1106,1199,1201],{"class":1200},"sFkSl"," '",[1106,1203,1204],{"class":1115},"@riavzon\u002Fshield-base",[1106,1206,1207],{"class":1200},"'",[1106,1209,1210],{"class":1186},";\n",[1106,1212,1214],{"class":1108,"line":1213},2,[1106,1215,1216],{"emptyLinePlaceholder":8},"\n",[1106,1218,1220],{"class":1108,"line":1219},3,[1106,1221,1223],{"class":1222},"sghk6","\u002F\u002F Compile with built-in providers only\n",[1106,1225,1227,1230,1233,1236,1238,1241,1243,1245,1247,1250,1252],{"class":1108,"line":1226},4,[1106,1228,1229],{"class":1182},"await",[1106,1231,1232],{"class":1112}," getCrawlersIps",[1106,1234,1235],{"class":1186},"(",[1106,1237,1207],{"class":1200},[1106,1239,1240],{"class":1115},".\u002Fout",[1106,1242,1207],{"class":1200},[1106,1244,899],{"class":1186},[1106,1246,1207],{"class":1200},[1106,1248,1249],{"class":1115},"mmdbctl",[1106,1251,1207],{"class":1200},[1106,1253,1254],{"class":1186},");\n",[1106,1256,1258],{"class":1108,"line":1257},5,[1106,1259,1216],{"emptyLinePlaceholder":8},[1106,1261,1263],{"class":1108,"line":1262},6,[1106,1264,1265],{"class":1222},"\u002F\u002F Or merge custom providers with built-in ones\n",[1106,1267,1269,1271,1274,1276,1279,1281,1283,1285,1287,1289],{"class":1108,"line":1268},7,[1106,1270,1183],{"class":1182},[1106,1272,1273],{"class":1182}," type",[1106,1275,1187],{"class":1186},[1106,1277,1278],{"class":1190},"ProvidersLists",[1106,1280,1194],{"class":1186},[1106,1282,1197],{"class":1182},[1106,1284,1201],{"class":1200},[1106,1286,1204],{"class":1115},[1106,1288,1207],{"class":1200},[1106,1290,1210],{"class":1186},[1106,1292,1294],{"class":1108,"line":1293},8,[1106,1295,1216],{"emptyLinePlaceholder":8},[1106,1297,1299,1303,1307,1311,1315,1318,1321],{"class":1108,"line":1298},9,[1106,1300,1302],{"class":1301},"sl46w","const",[1106,1304,1306],{"class":1305},"s3JHE"," customProviders",[1106,1308,1310],{"class":1309},"saOXh",":",[1106,1312,1314],{"class":1313},"sFs1U"," ProvidersLists",[1106,1316,1317],{"class":1186},"[] ",[1106,1319,1320],{"class":1309},"=",[1106,1322,1323],{"class":1186}," [\n",[1106,1325,1327],{"class":1108,"line":1326},10,[1106,1328,1329],{"class":1186},"  {\n",[1106,1331,1333,1336,1339,1341,1344,1346],{"class":1108,"line":1332},11,[1106,1334,1335],{"class":1190},"    name",[1106,1337,1310],{"class":1338},"s34zl",[1106,1340,1201],{"class":1200},[1106,1342,1343],{"class":1115},"cloudflare",[1106,1345,1207],{"class":1200},[1106,1347,1348],{"class":1186},",\n",[1106,1350,1352,1355,1357,1359,1361,1363],{"class":1108,"line":1351},12,[1106,1353,1354],{"class":1190},"    type",[1106,1356,1310],{"class":1338},[1106,1358,1201],{"class":1200},[1106,1360,889],{"class":1115},[1106,1362,1207],{"class":1200},[1106,1364,1348],{"class":1186},[1106,1366,1368,1371,1373],{"class":1108,"line":1367},13,[1106,1369,1370],{"class":1190},"    urls",[1106,1372,1310],{"class":1338},[1106,1374,1323],{"class":1186},[1106,1376,1378,1381,1384,1386],{"class":1108,"line":1377},14,[1106,1379,1380],{"class":1200},"      '",[1106,1382,1383],{"class":1115},"https:\u002F\u002Fwww.cloudflare.com\u002Fips-v4",[1106,1385,1207],{"class":1200},[1106,1387,1348],{"class":1186},[1106,1389,1391,1393,1396,1398],{"class":1108,"line":1390},15,[1106,1392,1380],{"class":1200},[1106,1394,1395],{"class":1115},"https:\u002F\u002Fwww.cloudflare.com\u002Fips-v6",[1106,1397,1207],{"class":1200},[1106,1399,1348],{"class":1186},[1106,1401,1403],{"class":1108,"line":1402},16,[1106,1404,1405],{"class":1186},"    ],\n",[1106,1407,1409],{"class":1108,"line":1408},17,[1106,1410,1411],{"class":1186},"  },\n",[1106,1413,1415],{"class":1108,"line":1414},18,[1106,1416,1417],{"class":1186},"];\n",[1106,1419,1421],{"class":1108,"line":1420},19,[1106,1422,1216],{"emptyLinePlaceholder":8},[1106,1424,1426,1428,1430,1432,1434,1436,1438,1440,1442,1444,1446,1448,1451],{"class":1108,"line":1425},20,[1106,1427,1229],{"class":1182},[1106,1429,1232],{"class":1112},[1106,1431,1235],{"class":1186},[1106,1433,1207],{"class":1200},[1106,1435,1240],{"class":1115},[1106,1437,1207],{"class":1200},[1106,1439,899],{"class":1186},[1106,1441,1207],{"class":1200},[1106,1443,1249],{"class":1115},[1106,1445,1207],{"class":1200},[1106,1447,899],{"class":1186},[1106,1449,1450],{"class":1190},"customProviders",[1106,1452,1254],{"class":1186},[1454,1455,1456],"note",{},[821,1457,1458,1459,1461],{},"When custom providers are passed, they are merged with the built-in datasets and compiled into a single ",[828,1460,841],{}," database.",[852,1463],{},[855,1465,1467],{"id":1466},"record-structure","Record Structure",[1096,1469,1471],{"className":1172,"code":1470,"language":1175,"meta":1102,"style":1102},"interface CrawlersRecord {\n  range: string;        \u002F\u002F IP prefix, e.g. \"66.249.66.0\u002F24\"\n  provider: string;     \u002F\u002F Provider name, e.g. \"google\", \"bing\", \"apple\"\n  syncToken: string;    \u002F\u002F Provider sync token (when available)\n  creationTime: string; \u002F\u002F Provider creation timestamp\n}\n\ninterface ProvidersLists {\n  name: string;                    \u002F\u002F Stored as the `provider` field in the database\n  type: 'HTML' | 'JSON' | 'CSV';  \u002F\u002F Format of the source URL\n  urls: string[];                  \u002F\u002F One or more URLs to fetch\n}\n",[828,1472,1473,1484,1500,1515,1530,1545,1550,1554,1562,1577,1613,1628],{"__ignoreMap":1102},[1106,1474,1475,1478,1481],{"class":1108,"line":1109},[1106,1476,1477],{"class":1301},"interface",[1106,1479,1480],{"class":1313}," CrawlersRecord",[1106,1482,1483],{"class":1186}," {\n",[1106,1485,1486,1489,1491,1494,1497],{"class":1108,"line":1213},[1106,1487,1488],{"class":1190},"  range",[1106,1490,1310],{"class":1309},[1106,1492,1493],{"class":1313}," string",[1106,1495,1496],{"class":1186},";        ",[1106,1498,1499],{"class":1222},"\u002F\u002F IP prefix, e.g. \"66.249.66.0\u002F24\"\n",[1106,1501,1502,1505,1507,1509,1512],{"class":1108,"line":1219},[1106,1503,1504],{"class":1190},"  provider",[1106,1506,1310],{"class":1309},[1106,1508,1493],{"class":1313},[1106,1510,1511],{"class":1186},";     ",[1106,1513,1514],{"class":1222},"\u002F\u002F Provider name, e.g. \"google\", \"bing\", \"apple\"\n",[1106,1516,1517,1520,1522,1524,1527],{"class":1108,"line":1226},[1106,1518,1519],{"class":1190},"  syncToken",[1106,1521,1310],{"class":1309},[1106,1523,1493],{"class":1313},[1106,1525,1526],{"class":1186},";    ",[1106,1528,1529],{"class":1222},"\u002F\u002F Provider sync token (when available)\n",[1106,1531,1532,1535,1537,1539,1542],{"class":1108,"line":1257},[1106,1533,1534],{"class":1190},"  creationTime",[1106,1536,1310],{"class":1309},[1106,1538,1493],{"class":1313},[1106,1540,1541],{"class":1186},"; ",[1106,1543,1544],{"class":1222},"\u002F\u002F Provider creation timestamp\n",[1106,1546,1547],{"class":1108,"line":1262},[1106,1548,1549],{"class":1186},"}\n",[1106,1551,1552],{"class":1108,"line":1268},[1106,1553,1216],{"emptyLinePlaceholder":8},[1106,1555,1556,1558,1560],{"class":1108,"line":1293},[1106,1557,1477],{"class":1301},[1106,1559,1314],{"class":1313},[1106,1561,1483],{"class":1186},[1106,1563,1564,1567,1569,1571,1574],{"class":1108,"line":1298},[1106,1565,1566],{"class":1190},"  name",[1106,1568,1310],{"class":1309},[1106,1570,1493],{"class":1313},[1106,1572,1573],{"class":1186},";                    ",[1106,1575,1576],{"class":1222},"\u002F\u002F Stored as the `provider` field in the database\n",[1106,1578,1579,1582,1584,1586,1588,1590,1593,1595,1597,1599,1601,1603,1605,1607,1610],{"class":1108,"line":1326},[1106,1580,1581],{"class":1190},"  type",[1106,1583,1310],{"class":1309},[1106,1585,1201],{"class":1200},[1106,1587,988],{"class":1115},[1106,1589,1207],{"class":1200},[1106,1591,1592],{"class":1309}," |",[1106,1594,1201],{"class":1200},[1106,1596,889],{"class":1115},[1106,1598,1207],{"class":1200},[1106,1600,1592],{"class":1309},[1106,1602,1201],{"class":1200},[1106,1604,1036],{"class":1115},[1106,1606,1207],{"class":1200},[1106,1608,1609],{"class":1186},";  ",[1106,1611,1612],{"class":1222},"\u002F\u002F Format of the source URL\n",[1106,1614,1615,1618,1620,1622,1625],{"class":1108,"line":1332},[1106,1616,1617],{"class":1190},"  urls",[1106,1619,1310],{"class":1309},[1106,1621,1493],{"class":1313},[1106,1623,1624],{"class":1186},"[];                  ",[1106,1626,1627],{"class":1222},"\u002F\u002F One or more URLs to fetch\n",[1106,1629,1630],{"class":1108,"line":1351},[1106,1631,1549],{"class":1186},[852,1633],{},[855,1635,1637],{"id":1636},"example-lookup","Example Lookup",[1096,1639,1642],{"className":1098,"code":1640,"filename":1641,"language":1101,"meta":1102,"style":1102},"mmdbctl read -f json-pretty 66.249.66.1 outputDirectory\u002FgoodBots.mmdb\n","Terminal",[828,1643,1644],{"__ignoreMap":1102},[1106,1645,1646,1648,1651,1654,1657,1661],{"class":1108,"line":1109},[1106,1647,1249],{"class":1112},[1106,1649,1650],{"class":1115}," read",[1106,1652,1653],{"class":1122}," -f",[1106,1655,1656],{"class":1115}," json-pretty",[1106,1658,1660],{"class":1659},"spgvN"," 66.249.66.1",[1106,1662,1663],{"class":1115}," outputDirectory\u002FgoodBots.mmdb\n",[1096,1665,1668],{"className":1666,"code":1667,"language":5,"meta":1102,"style":1102},"language-json shiki shiki-themes light-plus light-plus dracula","{\n  \"provider\": \"google\",\n  \"range\": \"66.249.66.0\u002F24\",\n  \"syncToken\": \"1710000000\",\n  \"creationTime\": \"2024-03-09T22:00:00.000Z\"\n}\n",[828,1669,1670,1675,1700,1719,1739,1758],{"__ignoreMap":1102},[1106,1671,1672],{"class":1108,"line":1109},[1106,1673,1674],{"class":1186},"{\n",[1106,1676,1677,1681,1685,1688,1690,1693,1696,1698],{"class":1108,"line":1213},[1106,1678,1680],{"class":1679},"saJyd","  \"",[1106,1682,1684],{"class":1683},"s_W10","provider",[1106,1686,1687],{"class":1679},"\"",[1106,1689,1310],{"class":1309},[1106,1691,1692],{"class":1200}," \"",[1106,1694,1695],{"class":1115},"google",[1106,1697,1687],{"class":1200},[1106,1699,1348],{"class":1186},[1106,1701,1702,1704,1706,1708,1710,1712,1715,1717],{"class":1108,"line":1219},[1106,1703,1680],{"class":1679},[1106,1705,741],{"class":1683},[1106,1707,1687],{"class":1679},[1106,1709,1310],{"class":1309},[1106,1711,1692],{"class":1200},[1106,1713,1714],{"class":1115},"66.249.66.0\u002F24",[1106,1716,1687],{"class":1200},[1106,1718,1348],{"class":1186},[1106,1720,1721,1723,1726,1728,1730,1732,1735,1737],{"class":1108,"line":1226},[1106,1722,1680],{"class":1679},[1106,1724,1725],{"class":1683},"syncToken",[1106,1727,1687],{"class":1679},[1106,1729,1310],{"class":1309},[1106,1731,1692],{"class":1200},[1106,1733,1734],{"class":1115},"1710000000",[1106,1736,1687],{"class":1200},[1106,1738,1348],{"class":1186},[1106,1740,1741,1743,1746,1748,1750,1752,1755],{"class":1108,"line":1257},[1106,1742,1680],{"class":1679},[1106,1744,1745],{"class":1683},"creationTime",[1106,1747,1687],{"class":1679},[1106,1749,1310],{"class":1309},[1106,1751,1692],{"class":1200},[1106,1753,1754],{"class":1115},"2024-03-09T22:00:00.000Z",[1106,1756,1757],{"class":1200},"\"\n",[1106,1759,1760],{"class":1108,"line":1262},[1106,1761,1549],{"class":1186},[1763,1764,1765],"caution",{},[821,1766,1767,1768,1771,1772,1777],{},"The type field is a special field that the success of the data retrieval depends on.\nIf the links you are providing include a regular html\u002Fmarkdown\u002Fother-raw-text-data page, use HTML. If it is a link to a CSV file, use CSV.\nIf it is a JSON (e.g., ",[893,1769,895],{"href":895,"rel":1770},[897],"), use JSON.\nProviding urls that mixes CSV with JSON data or raw text with CSV and JSON will fail to process this provider.\nVisit the built-in providers to get an idea of the parsing engine or check the ",[893,1773,1776],{"href":1774,"rel":1775},"https:\u002F\u002Fgithub.com\u002FSergo706\u002Fshield-base-cli\u002Ftree\u002Fmain\u002Fsrc\u002Fscripts\u002FgoodBotsScrapper",[897],"source code",".",[1779,1780,1781],"style",{},"html pre.shiki code .sHOzp, html code.shiki .sHOzp{--shiki-light:#795E26;--shiki-default:#795E26;--shiki-dark:#50FA7B}html pre.shiki code .sFB1V, html code.shiki .sFB1V{--shiki-light:#A31515;--shiki-default:#A31515;--shiki-dark:#F1FA8C}html pre.shiki code .sjR7W, html code.shiki .sjR7W{--shiki-light:#0000FF;--shiki-default:#0000FF;--shiki-dark:#BD93F9}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZ328, html code.shiki .sZ328{--shiki-light:#AF00DB;--shiki-default:#AF00DB;--shiki-dark:#FF79C6}html pre.shiki code .sDd4n, html code.shiki .sDd4n{--shiki-light:#000000;--shiki-default:#000000;--shiki-dark:#F8F8F2}html pre.shiki code .sjsA6, html code.shiki .sjsA6{--shiki-light:#001080;--shiki-default:#001080;--shiki-dark:#F8F8F2}html pre.shiki code .sFkSl, html code.shiki .sFkSl{--shiki-light:#A31515;--shiki-default:#A31515;--shiki-dark:#E9F284}html pre.shiki code .sghk6, html code.shiki .sghk6{--shiki-light:#008000;--shiki-default:#008000;--shiki-dark:#6272A4}html pre.shiki code .sl46w, html code.shiki .sl46w{--shiki-light:#0000FF;--shiki-default:#0000FF;--shiki-dark:#FF79C6}html pre.shiki code .s3JHE, html code.shiki .s3JHE{--shiki-light:#0070C1;--shiki-default:#0070C1;--shiki-dark:#F8F8F2}html pre.shiki code .saOXh, html code.shiki .saOXh{--shiki-light:#000000;--shiki-default:#000000;--shiki-dark:#FF79C6}html pre.shiki code .sFs1U, html code.shiki .sFs1U{--shiki-light:#267F99;--shiki-light-font-style:inherit;--shiki-default:#267F99;--shiki-default-font-style:inherit;--shiki-dark:#8BE9FD;--shiki-dark-font-style:italic}html pre.shiki code .s34zl, html code.shiki .s34zl{--shiki-light:#001080;--shiki-default:#001080;--shiki-dark:#FF79C6}html pre.shiki code .spgvN, html code.shiki .spgvN{--shiki-light:#098658;--shiki-default:#098658;--shiki-dark:#BD93F9}html pre.shiki code .saJyd, html code.shiki .saJyd{--shiki-light:#0451A5;--shiki-default:#0451A5;--shiki-dark:#8BE9FE}html pre.shiki code .s_W10, html code.shiki .s_W10{--shiki-light:#0451A5;--shiki-default:#0451A5;--shiki-dark:#8BE9FD}",{"title":1102,"searchDepth":1213,"depth":1213,"links":1783},[1784,1785,1786,1787],{"id":857,"depth":1213,"text":858},{"id":1090,"depth":1213,"text":1091},{"id":1466,"depth":1213,"text":1467},{"id":1636,"depth":1213,"text":1637},"IP ranges for major search engine and platform crawlers fetched from official provider geofeeds, compiled into goodBots.mmdb.","md","i-lucide-search",{},null,"---\ntitle: Verified Crawlers\ndescription: IP ranges for major search engine and platform crawlers fetched from official provider geofeeds, compiled into goodBots.mmdb.\nicon: i-lucide-search\n---\n\nThe Crawlers source fetches IP ranges for major search engine crawlers and legitimate automated agents from their official geofeeds and documentation pages. Data is extracted from JSON, CSV, and HTML sources, then merged and compiled into a single MMDB database.\n\nThe fetcher uses a tiered fetch with a fallback mechanism that invokes `curl` to bypass anti-scraping measures on social media geofeeds when it detects a regular fetch being blocked.\n\n**Output file:** `goodBots.mmdb`\n\n::warning\nMake sure `curl` is installed on your system when using this data source. The fallback mechanism requires it to fetch certain provider pages.\n::\n\n---\n\n## Built-in Providers\n\n| Provider | Type | Source |\n| --- | --- | --- |\n| Google | JSON | [googlebot.json](https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fgooglebot.json), [special-crawlers.json](https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fspecial-crawlers.json), [user-triggered-fetchers.json](https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fuser-triggered-fetchers.json), [user-triggered-fetchers-google.json](https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fuser-triggered-fetchers-google.json), [goog.json](https:\u002F\u002Fwww.gstatic.com\u002Fipranges\u002Fgoog.json) |\n| Bing | JSON | [bingbot.json](https:\u002F\u002Fwww.bing.com\u002Ftoolbox\u002Fbingbot.json) |\n| OpenAI | JSON | [gptbot.json](https:\u002F\u002Fopenai.com\u002Fgptbot.json), [searchbot.json](https:\u002F\u002Fopenai.com\u002Fsearchbot.json) |\n| Apple | JSON | [applebot.json](https:\u002F\u002Fsearch.developer.apple.com\u002Fapplebot.json) |\n| Ahrefs | JSON | [crawler-ip-ranges](https:\u002F\u002Fapi.ahrefs.com\u002Fv3\u002Fpublic\u002Fcrawler-ip-ranges) |\n| DuckDuckGo | HTML | [duckassistbot](https:\u002F\u002Fduckduckgo.com\u002Fduckduckgo-help-pages\u002Fresults\u002Fduckassistbot), [duckduckbot](https:\u002F\u002Fduckduckgo.com\u002Fduckduckgo-help-pages\u002Fresults\u002Fduckduckbot) |\n| Common Crawl | HTML | [commoncrawl.org\u002Ffaq](https:\u002F\u002Fcommoncrawl.org\u002Ffaq) |\n| X \u002F Twitter | HTML | [troubleshooting-cards](https:\u002F\u002Fdeveloper.x.com\u002Fen\u002Fdocs\u002Fx-for-websites\u002Fcards\u002Fguides\u002Ftroubleshooting-cards) |\n| Facebook | CSV | [facebook.com\u002Fpeering\u002Fgeofeed](https:\u002F\u002Fwww.facebook.com\u002Fpeering\u002Fgeofeed) |\n| Pinterest | HTML | [pinterestbot](https:\u002F\u002Fhelp.pinterest.com\u002Fen\u002Fbusiness\u002Farticle\u002Fpinterestbot) |\n| Telegram | HTML | [bots\u002Fwebhooks](https:\u002F\u002Fcore.telegram.org\u002Fbots\u002Fwebhooks) |\n| Semrush | HTML | [semrush.com\u002Fkb\u002F1149](https:\u002F\u002Fwww.semrush.com\u002Fkb\u002F1149-issues-with-crawling-a-domain) |\n\n---\n\n## Usage\n\n::code-group\n```bash [pnpm]\npnpm dlx @riavzon\u002Fshield-base --seo\n```\n\n```bash [yarn]\nyarn dlx @riavzon\u002Fshield-base --seo\n```\n\n```bash [npm]\nnpx @riavzon\u002Fshield-base --seo\n```\n\n```bash [bun]\nbunx @riavzon\u002Fshield-base --seo\n```\n```ts [Programmatic]\nimport { getCrawlersIps } from '@riavzon\u002Fshield-base';\n\n\u002F\u002F Compile with built-in providers only\nawait getCrawlersIps('.\u002Fout', 'mmdbctl');\n\n\u002F\u002F Or merge custom providers with built-in ones\nimport type { ProvidersLists } from '@riavzon\u002Fshield-base';\n\nconst customProviders: ProvidersLists[] = [\n  {\n    name: 'cloudflare',\n    type: 'JSON',\n    urls: [\n      'https:\u002F\u002Fwww.cloudflare.com\u002Fips-v4',\n      'https:\u002F\u002Fwww.cloudflare.com\u002Fips-v6',\n    ],\n  },\n];\n\nawait getCrawlersIps('.\u002Fout', 'mmdbctl', customProviders);\n```\n::\n\n::note\nWhen custom providers are passed, they are merged with the built-in datasets and compiled into a single `goodBots.mmdb` database.\n::\n\n---\n\n## Record Structure\n\n```ts\ninterface CrawlersRecord {\n  range: string;        \u002F\u002F IP prefix, e.g. \"66.249.66.0\u002F24\"\n  provider: string;     \u002F\u002F Provider name, e.g. \"google\", \"bing\", \"apple\"\n  syncToken: string;    \u002F\u002F Provider sync token (when available)\n  creationTime: string; \u002F\u002F Provider creation timestamp\n}\n\ninterface ProvidersLists {\n  name: string;                    \u002F\u002F Stored as the `provider` field in the database\n  type: 'HTML' | 'JSON' | 'CSV';  \u002F\u002F Format of the source URL\n  urls: string[];                  \u002F\u002F One or more URLs to fetch\n}\n```\n\n---\n\n## Example Lookup\n\n```bash [Terminal]\nmmdbctl read -f json-pretty 66.249.66.1 outputDirectory\u002FgoodBots.mmdb\n```\n\n```json\n{\n  \"provider\": \"google\",\n  \"range\": \"66.249.66.0\u002F24\",\n  \"syncToken\": \"1710000000\",\n  \"creationTime\": \"2024-03-09T22:00:00.000Z\"\n}\n```\n::caution\nThe type field is a special field that the success of the data retrieval depends on.\n  If the links you are providing include a regular html\u002Fmarkdown\u002Fother-raw-text-data page, use HTML. If it is a link to a CSV file, use CSV.\n  If it is a JSON (e.g., https:\u002F\u002Fdevelopers.google.com\u002Fstatic\u002Fsearch\u002Fapis\u002Fipranges\u002Fgooglebot.json), use JSON.\n  Providing urls that mixes CSV with JSON data or raw text with CSV and JSON will fail to process this provider.\n  Visit the built-in providers to get an idea of the parsing engine or check the [source code](https:\u002F\u002Fgithub.com\u002FSergo706\u002Fshield-base-cli\u002Ftree\u002Fmain\u002Fsrc\u002Fscripts\u002FgoodBotsScrapper).\n::",{"title":99,"description":1788},"77gnWVEY8_XdFkEjmkL7H9pD0u_bF27JLtNSOFSE1Gk",[1797,1798],{"title":95,"path":96,"stem":97,"children":-1},{"title":103,"path":104,"stem":105,"children":-1},{"id":816,"title":99,"body":1800,"description":1788,"extension":1789,"icon":1790,"meta":2524,"module":1792,"navigation":8,"path":100,"rawbody":1793,"seo":2525,"stem":101,"__hash__":1795},{"type":818,"value":1801,"toc":2518},[1802,1804,1808,1814,1820,1822,1824,1988,1990,1992,2256,2262,2264,2266,2402,2404,2406,2424,2506,2516],[821,1803,823],{},[821,1805,826,1806,831],{},[828,1807,830],{},[821,1809,1810,838,1812],{},[835,1811,837],{},[828,1813,841],{},[843,1815,1816],{},[821,1817,847,1818,850],{},[828,1819,830],{},[852,1821],{},[855,1823,858],{"id":857},[860,1825,1826,1836],{},[863,1827,1828],{},[866,1829,1830,1832,1834],{},[869,1831,871],{},[869,1833,874],{},[869,1835,877],{},[879,1837,1838,1861,1872,1886,1897,1908,1922,1933,1944,1955,1966,1977],{},[866,1839,1840,1842,1844],{},[884,1841,886],{},[884,1843,889],{},[884,1845,1846,899,1849,899,1852,899,1855,899,1858],{},[893,1847,898],{"href":895,"rel":1848},[897],[893,1850,904],{"href":902,"rel":1851},[897],[893,1853,909],{"href":907,"rel":1854},[897],[893,1856,914],{"href":912,"rel":1857},[897],[893,1859,919],{"href":917,"rel":1860},[897],[866,1862,1863,1865,1867],{},[884,1864,924],{},[884,1866,889],{},[884,1868,1869],{},[893,1870,933],{"href":931,"rel":1871},[897],[866,1873,1874,1876,1878],{},[884,1875,938],{},[884,1877,889],{},[884,1879,1880,899,1883],{},[893,1881,947],{"href":945,"rel":1882},[897],[893,1884,952],{"href":950,"rel":1885},[897],[866,1887,1888,1890,1892],{},[884,1889,957],{},[884,1891,889],{},[884,1893,1894],{},[893,1895,966],{"href":964,"rel":1896},[897],[866,1898,1899,1901,1903],{},[884,1900,971],{},[884,1902,889],{},[884,1904,1905],{},[893,1906,980],{"href":978,"rel":1907},[897],[866,1909,1910,1912,1914],{},[884,1911,985],{},[884,1913,988],{},[884,1915,1916,899,1919],{},[893,1917,995],{"href":993,"rel":1918},[897],[893,1920,1000],{"href":998,"rel":1921},[897],[866,1923,1924,1926,1928],{},[884,1925,1005],{},[884,1927,988],{},[884,1929,1930],{},[893,1931,1014],{"href":1012,"rel":1932},[897],[866,1934,1935,1937,1939],{},[884,1936,1019],{},[884,1938,988],{},[884,1940,1941],{},[893,1942,1028],{"href":1026,"rel":1943},[897],[866,1945,1946,1948,1950],{},[884,1947,1033],{},[884,1949,1036],{},[884,1951,1952],{},[893,1953,1043],{"href":1041,"rel":1954},[897],[866,1956,1957,1959,1961],{},[884,1958,1048],{},[884,1960,988],{},[884,1962,1963],{},[893,1964,1057],{"href":1055,"rel":1965},[897],[866,1967,1968,1970,1972],{},[884,1969,1062],{},[884,1971,988],{},[884,1973,1974],{},[893,1975,1071],{"href":1069,"rel":1976},[897],[866,1978,1979,1981,1983],{},[884,1980,1076],{},[884,1982,988],{},[884,1984,1985],{},[893,1986,1085],{"href":1083,"rel":1987},[897],[852,1989],{},[855,1991,1091],{"id":1090},[1093,1993,1994,2008,2022,2034,2046],{},[1096,1995,1996],{"className":1098,"code":1099,"filename":1100,"language":1101,"meta":1102,"style":1102},[828,1997,1998],{"__ignoreMap":1102},[1106,1999,2000,2002,2004,2006],{"class":1108,"line":1109},[1106,2001,1100],{"class":1112},[1106,2003,1116],{"class":1115},[1106,2005,1119],{"class":1115},[1106,2007,1123],{"class":1122},[1096,2009,2010],{"className":1098,"code":1126,"filename":1127,"language":1101,"meta":1102,"style":1102},[828,2011,2012],{"__ignoreMap":1102},[1106,2013,2014,2016,2018,2020],{"class":1108,"line":1109},[1106,2015,1127],{"class":1112},[1106,2017,1116],{"class":1115},[1106,2019,1119],{"class":1115},[1106,2021,1123],{"class":1122},[1096,2023,2024],{"className":1098,"code":1142,"filename":1143,"language":1101,"meta":1102,"style":1102},[828,2025,2026],{"__ignoreMap":1102},[1106,2027,2028,2030,2032],{"class":1108,"line":1109},[1106,2029,1150],{"class":1112},[1106,2031,1119],{"class":1115},[1106,2033,1123],{"class":1122},[1096,2035,2036],{"className":1098,"code":1157,"filename":1158,"language":1101,"meta":1102,"style":1102},[828,2037,2038],{"__ignoreMap":1102},[1106,2039,2040,2042,2044],{"class":1108,"line":1109},[1106,2041,1165],{"class":1112},[1106,2043,1119],{"class":1115},[1106,2045,1123],{"class":1122},[1096,2047,2048],{"className":1172,"code":1173,"filename":1174,"language":1175,"meta":1102,"style":1102},[828,2049,2050,2070,2074,2078,2102,2106,2110,2132,2136,2152,2156,2170,2184,2192,2202,2212,2216,2220,2224,2228],{"__ignoreMap":1102},[1106,2051,2052,2054,2056,2058,2060,2062,2064,2066,2068],{"class":1108,"line":1109},[1106,2053,1183],{"class":1182},[1106,2055,1187],{"class":1186},[1106,2057,1191],{"class":1190},[1106,2059,1194],{"class":1186},[1106,2061,1197],{"class":1182},[1106,2063,1201],{"class":1200},[1106,2065,1204],{"class":1115},[1106,2067,1207],{"class":1200},[1106,2069,1210],{"class":1186},[1106,2071,2072],{"class":1108,"line":1213},[1106,2073,1216],{"emptyLinePlaceholder":8},[1106,2075,2076],{"class":1108,"line":1219},[1106,2077,1223],{"class":1222},[1106,2079,2080,2082,2084,2086,2088,2090,2092,2094,2096,2098,2100],{"class":1108,"line":1226},[1106,2081,1229],{"class":1182},[1106,2083,1232],{"class":1112},[1106,2085,1235],{"class":1186},[1106,2087,1207],{"class":1200},[1106,2089,1240],{"class":1115},[1106,2091,1207],{"class":1200},[1106,2093,899],{"class":1186},[1106,2095,1207],{"class":1200},[1106,2097,1249],{"class":1115},[1106,2099,1207],{"class":1200},[1106,2101,1254],{"class":1186},[1106,2103,2104],{"class":1108,"line":1257},[1106,2105,1216],{"emptyLinePlaceholder":8},[1106,2107,2108],{"class":1108,"line":1262},[1106,2109,1265],{"class":1222},[1106,2111,2112,2114,2116,2118,2120,2122,2124,2126,2128,2130],{"class":1108,"line":1268},[1106,2113,1183],{"class":1182},[1106,2115,1273],{"class":1182},[1106,2117,1187],{"class":1186},[1106,2119,1278],{"class":1190},[1106,2121,1194],{"class":1186},[1106,2123,1197],{"class":1182},[1106,2125,1201],{"class":1200},[1106,2127,1204],{"class":1115},[1106,2129,1207],{"class":1200},[1106,2131,1210],{"class":1186},[1106,2133,2134],{"class":1108,"line":1293},[1106,2135,1216],{"emptyLinePlaceholder":8},[1106,2137,2138,2140,2142,2144,2146,2148,2150],{"class":1108,"line":1298},[1106,2139,1302],{"class":1301},[1106,2141,1306],{"class":1305},[1106,2143,1310],{"class":1309},[1106,2145,1314],{"class":1313},[1106,2147,1317],{"class":1186},[1106,2149,1320],{"class":1309},[1106,2151,1323],{"class":1186},[1106,2153,2154],{"class":1108,"line":1326},[1106,2155,1329],{"class":1186},[1106,2157,2158,2160,2162,2164,2166,2168],{"class":1108,"line":1332},[1106,2159,1335],{"class":1190},[1106,2161,1310],{"class":1338},[1106,2163,1201],{"class":1200},[1106,2165,1343],{"class":1115},[1106,2167,1207],{"class":1200},[1106,2169,1348],{"class":1186},[1106,2171,2172,2174,2176,2178,2180,2182],{"class":1108,"line":1351},[1106,2173,1354],{"class":1190},[1106,2175,1310],{"class":1338},[1106,2177,1201],{"class":1200},[1106,2179,889],{"class":1115},[1106,2181,1207],{"class":1200},[1106,2183,1348],{"class":1186},[1106,2185,2186,2188,2190],{"class":1108,"line":1367},[1106,2187,1370],{"class":1190},[1106,2189,1310],{"class":1338},[1106,2191,1323],{"class":1186},[1106,2193,2194,2196,2198,2200],{"class":1108,"line":1377},[1106,2195,1380],{"class":1200},[1106,2197,1383],{"class":1115},[1106,2199,1207],{"class":1200},[1106,2201,1348],{"class":1186},[1106,2203,2204,2206,2208,2210],{"class":1108,"line":1390},[1106,2205,1380],{"class":1200},[1106,2207,1395],{"class":1115},[1106,2209,1207],{"class":1200},[1106,2211,1348],{"class":1186},[1106,2213,2214],{"class":1108,"line":1402},[1106,2215,1405],{"class":1186},[1106,2217,2218],{"class":1108,"line":1408},[1106,2219,1411],{"class":1186},[1106,2221,2222],{"class":1108,"line":1414},[1106,2223,1417],{"class":1186},[1106,2225,2226],{"class":1108,"line":1420},[1106,2227,1216],{"emptyLinePlaceholder":8},[1106,2229,2230,2232,2234,2236,2238,2240,2242,2244,2246,2248,2250,2252,2254],{"class":1108,"line":1425},[1106,2231,1229],{"class":1182},[1106,2233,1232],{"class":1112},[1106,2235,1235],{"class":1186},[1106,2237,1207],{"class":1200},[1106,2239,1240],{"class":1115},[1106,2241,1207],{"class":1200},[1106,2243,899],{"class":1186},[1106,2245,1207],{"class":1200},[1106,2247,1249],{"class":1115},[1106,2249,1207],{"class":1200},[1106,2251,899],{"class":1186},[1106,2253,1450],{"class":1190},[1106,2255,1254],{"class":1186},[1454,2257,2258],{},[821,2259,1458,2260,1461],{},[828,2261,841],{},[852,2263],{},[855,2265,1467],{"id":1466},[1096,2267,2268],{"className":1172,"code":1470,"language":1175,"meta":1102,"style":1102},[828,2269,2270,2278,2290,2302,2314,2326,2330,2334,2342,2354,2386,2398],{"__ignoreMap":1102},[1106,2271,2272,2274,2276],{"class":1108,"line":1109},[1106,2273,1477],{"class":1301},[1106,2275,1480],{"class":1313},[1106,2277,1483],{"class":1186},[1106,2279,2280,2282,2284,2286,2288],{"class":1108,"line":1213},[1106,2281,1488],{"class":1190},[1106,2283,1310],{"class":1309},[1106,2285,1493],{"class":1313},[1106,2287,1496],{"class":1186},[1106,2289,1499],{"class":1222},[1106,2291,2292,2294,2296,2298,2300],{"class":1108,"line":1219},[1106,2293,1504],{"class":1190},[1106,2295,1310],{"class":1309},[1106,2297,1493],{"class":1313},[1106,2299,1511],{"class":1186},[1106,2301,1514],{"class":1222},[1106,2303,2304,2306,2308,2310,2312],{"class":1108,"line":1226},[1106,2305,1519],{"class":1190},[1106,2307,1310],{"class":1309},[1106,2309,1493],{"class":1313},[1106,2311,1526],{"class":1186},[1106,2313,1529],{"class":1222},[1106,2315,2316,2318,2320,2322,2324],{"class":1108,"line":1257},[1106,2317,1534],{"class":1190},[1106,2319,1310],{"class":1309},[1106,2321,1493],{"class":1313},[1106,2323,1541],{"class":1186},[1106,2325,1544],{"class":1222},[1106,2327,2328],{"class":1108,"line":1262},[1106,2329,1549],{"class":1186},[1106,2331,2332],{"class":1108,"line":1268},[1106,2333,1216],{"emptyLinePlaceholder":8},[1106,2335,2336,2338,2340],{"class":1108,"line":1293},[1106,2337,1477],{"class":1301},[1106,2339,1314],{"class":1313},[1106,2341,1483],{"class":1186},[1106,2343,2344,2346,2348,2350,2352],{"class":1108,"line":1298},[1106,2345,1566],{"class":1190},[1106,2347,1310],{"class":1309},[1106,2349,1493],{"class":1313},[1106,2351,1573],{"class":1186},[1106,2353,1576],{"class":1222},[1106,2355,2356,2358,2360,2362,2364,2366,2368,2370,2372,2374,2376,2378,2380,2382,2384],{"class":1108,"line":1326},[1106,2357,1581],{"class":1190},[1106,2359,1310],{"class":1309},[1106,2361,1201],{"class":1200},[1106,2363,988],{"class":1115},[1106,2365,1207],{"class":1200},[1106,2367,1592],{"class":1309},[1106,2369,1201],{"class":1200},[1106,2371,889],{"class":1115},[1106,2373,1207],{"class":1200},[1106,2375,1592],{"class":1309},[1106,2377,1201],{"class":1200},[1106,2379,1036],{"class":1115},[1106,2381,1207],{"class":1200},[1106,2383,1609],{"class":1186},[1106,2385,1612],{"class":1222},[1106,2387,2388,2390,2392,2394,2396],{"class":1108,"line":1332},[1106,2389,1617],{"class":1190},[1106,2391,1310],{"class":1309},[1106,2393,1493],{"class":1313},[1106,2395,1624],{"class":1186},[1106,2397,1627],{"class":1222},[1106,2399,2400],{"class":1108,"line":1351},[1106,2401,1549],{"class":1186},[852,2403],{},[855,2405,1637],{"id":1636},[1096,2407,2408],{"className":1098,"code":1640,"filename":1641,"language":1101,"meta":1102,"style":1102},[828,2409,2410],{"__ignoreMap":1102},[1106,2411,2412,2414,2416,2418,2420,2422],{"class":1108,"line":1109},[1106,2413,1249],{"class":1112},[1106,2415,1650],{"class":1115},[1106,2417,1653],{"class":1122},[1106,2419,1656],{"class":1115},[1106,2421,1660],{"class":1659},[1106,2423,1663],{"class":1115},[1096,2425,2426],{"className":1666,"code":1667,"language":5,"meta":1102,"style":1102},[828,2427,2428,2432,2450,2468,2486,2502],{"__ignoreMap":1102},[1106,2429,2430],{"class":1108,"line":1109},[1106,2431,1674],{"class":1186},[1106,2433,2434,2436,2438,2440,2442,2444,2446,2448],{"class":1108,"line":1213},[1106,2435,1680],{"class":1679},[1106,2437,1684],{"class":1683},[1106,2439,1687],{"class":1679},[1106,2441,1310],{"class":1309},[1106,2443,1692],{"class":1200},[1106,2445,1695],{"class":1115},[1106,2447,1687],{"class":1200},[1106,2449,1348],{"class":1186},[1106,2451,2452,2454,2456,2458,2460,2462,2464,2466],{"class":1108,"line":1219},[1106,2453,1680],{"class":1679},[1106,2455,741],{"class":1683},[1106,2457,1687],{"class":1679},[1106,2459,1310],{"class":1309},[1106,2461,1692],{"class":1200},[1106,2463,1714],{"class":1115},[1106,2465,1687],{"class":1200},[1106,2467,1348],{"class":1186},[1106,2469,2470,2472,2474,2476,2478,2480,2482,2484],{"class":1108,"line":1226},[1106,2471,1680],{"class":1679},[1106,2473,1725],{"class":1683},[1106,2475,1687],{"class":1679},[1106,2477,1310],{"class":1309},[1106,2479,1692],{"class":1200},[1106,2481,1734],{"class":1115},[1106,2483,1687],{"class":1200},[1106,2485,1348],{"class":1186},[1106,2487,2488,2490,2492,2494,2496,2498,2500],{"class":1108,"line":1257},[1106,2489,1680],{"class":1679},[1106,2491,1745],{"class":1683},[1106,2493,1687],{"class":1679},[1106,2495,1310],{"class":1309},[1106,2497,1692],{"class":1200},[1106,2499,1754],{"class":1115},[1106,2501,1757],{"class":1200},[1106,2503,2504],{"class":1108,"line":1262},[1106,2505,1549],{"class":1186},[1763,2507,2508],{},[821,2509,1767,2510,1771,2513,1777],{},[893,2511,895],{"href":895,"rel":2512},[897],[893,2514,1776],{"href":1774,"rel":2515},[897],[1779,2517,1781],{},{"title":1102,"searchDepth":1213,"depth":1213,"links":2519},[2520,2521,2522,2523],{"id":857,"depth":1213,"text":858},{"id":1090,"depth":1213,"text":1091},{"id":1466,"depth":1213,"text":1467},{"id":1636,"depth":1213,"text":1637},{},{"title":99,"description":1788},1780436288349]