# robots.txt for foto4art.nl # Optimized for crawl budget - blocks tracking params, faceted nav, analytics proxy # Last updated: 2026-03-24 User-agent: * # Allow static assets Allow: */modules/*.css Allow: */modules/*.js Allow: */modules/*.png Allow: */modules/*.jpg Allow: /js/jquery/* # ============================================ # ANALYTICS / TRACKING PARAMETERS # Google Analytics proxy - biggest crawl budget waste # ============================================ Disallow: /maql/ Disallow: /*?gtp= Disallow: /*>p= Disallow: /*?site_speed_sample_rate= Disallow: /*&site_speed_sample_rate= Disallow: /*?anonymize_ip= Disallow: /*&anonymize_ip= Disallow: /*?link_attribution= Disallow: /*&link_attribution= Disallow: /*?flid= Disallow: /*&flid= Disallow: /*?fn= Disallow: /*&fn= Disallow: /*?nsi= Disallow: /*&nsi= Disallow: /*?ss= Disallow: /*&ss= Disallow: /*?ae= Disallow: /*&ae= Disallow: /*?page_views= Disallow: /*&page_views= Disallow: /*?gtm= Disallow: /*>m= Disallow: /*?tid= Disallow: /*&tid= # UTM / campaign tracking Disallow: /*?utm_source= Disallow: /*&utm_source= Disallow: /*?utm_medium= Disallow: /*&utm_medium= Disallow: /*?utm_campaign= Disallow: /*&utm_campaign= Disallow: /*?utm_content= Disallow: /*&utm_content= Disallow: /*?utm_term= Disallow: /*&utm_term= # ============================================ # PRESTASHOP PARAMETERS - sorting, filtering, sessions # ============================================ Disallow: /*?order= Disallow: /*&order= Disallow: /*?tag= Disallow: /*&tag= Disallow: /*?id_currency= Disallow: /*&id_currency= Disallow: /*?search_query= Disallow: /*&search_query= Disallow: /*?back= Disallow: /*&back= Disallow: /*?n= Disallow: /*&n= Disallow: /*?p= Disallow: /*&p= Disallow: /*?id_product_attribute= Disallow: /*&id_product_attribute= Disallow: /*?resultsPerPage= Disallow: /*&resultsPerPage= # Faceted search / layered navigation Disallow: /*?q= Disallow: /*&q= Disallow: /*?selected_filters= Disallow: /*&selected_filters= # ============================================ # PRESTASHOP CONTROLLERS - private/user pages # ============================================ Disallow: /*controller=addresses Disallow: /*controller=address Disallow: /*controller=authentication Disallow: /*controller=cart Disallow: /*controller=discount Disallow: /*controller=footer Disallow: /*controller=get-file Disallow: /*controller=header Disallow: /*controller=history Disallow: /*controller=identity Disallow: /*controller=images.inc Disallow: /*controller=init Disallow: /*controller=my-account Disallow: /*controller=order Disallow: /*controller=order-slip Disallow: /*controller=order-detail Disallow: /*controller=order-follow Disallow: /*controller=order-return Disallow: /*controller=order-confirmation Disallow: /*controller=pagination Disallow: /*controller=password Disallow: /*controller=pdf-invoice Disallow: /*controller=pdf-order-return Disallow: /*controller=pdf-order-slip Disallow: /*controller=product-sort Disallow: /*controller=search Disallow: /*controller=statistics Disallow: /*controller=attachment Disallow: /*controller=guest-tracking # ============================================ # DIRECTORIES - internal / non-public # ============================================ Disallow: /crop-preview/ Disallow: /img-preview/ Disallow: /visual/ Disallow: /maql/ Disallow: /app/ Disallow: /cache/ Disallow: /classes/ Disallow: /config/ Disallow: /controllers/ Disallow: /download/ Disallow: /js/ Disallow: /localization/ Disallow: /log/ Disallow: /mails/ Disallow: /modules/ Disallow: /override/ Disallow: /pdf/ Disallow: /src/ Disallow: /tools/ Disallow: /translations/ Disallow: /upload/ Disallow: /var/ Disallow: /vendor/ Disallow: /webservice/ Disallow: /nl/app/ Disallow: /nl/cache/ Disallow: /nl/classes/ Disallow: /nl/config/ Disallow: /nl/controllers/ Disallow: /nl/download/ Disallow: /nl/js/ Disallow: /nl/localization/ Disallow: /nl/log/ Disallow: /nl/mails/ Disallow: /nl/modules/ Disallow: /nl/override/ Disallow: /nl/pdf/ Disallow: /nl/src/ Disallow: /nl/tools/ Disallow: /nl/translations/ Disallow: /nl/upload/ Disallow: /nl/var/ Disallow: /nl/vendor/ Disallow: /nl/webservice/ # ============================================ # USER ACCOUNT PAGES (language prefix) # Friendly URLs - login/password (not covered by controller blocks) Disallow: /aanmelden Disallow: /wachtwoord-opvragen # ============================================ Disallow: /*nl/password-recovery Disallow: /*nl/address Disallow: /*nl/addresses Disallow: /*nl/login Disallow: /*nl/cart Disallow: /*nl/discount Disallow: /*nl/order-history Disallow: /*nl/identity Disallow: /*nl/my-account Disallow: /*nl/order-follow Disallow: /*nl/credit-slip Disallow: /*nl/order Disallow: /*nl/search Disallow: /*nl/guest-tracking Disallow: /*nl/order-confirmation # ============================================ # BLOCK SEO SCRAPERS # ============================================ User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: SEOkicks Disallow: / User-agent: BLEXBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: Bytespider Disallow: / # Baidu ZABLOKOWANY (decyzja Agi 2026-10-10: chinska wyszukiwarka, poza rynkiem; do 276 tys. zadan/dobe). Dotyczy tez Baiduspider-render. claude Aga MB User-agent: Baiduspider Disallow: / # Asystenci AI — WPUSZCZAMY (decyzja Agi 2026-08-17, tak jak na nikkel-art). # Chcemy byc widoczni w odpowiedziach ChatGPT, Claude'a i pozostalych. # Scrapery SEO i Bytespider zostaja zablokowane nizej/wyzej — to co innego. User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: CCBot Allow: / User-agent: anthropic-ai Allow: / User-agent: Claude-Web Allow: / User-agent: Applebot-Extended Disallow: / # ============================================ # GOOGLE R&D CRAWLER (BLOCK) # ============================================ # GoogleOther to crawler ogolnego przeznaczenia zespolow Google (badania/rozwoj). # NIE indeksuje do wyszukiwarki — to robi Googlebot, ktory zostaje wpuszczony — # i NIE jest wymagany przez Merchant Center. # Pomiar 2026-08-16 (pelna doba, logi dostepu): 65 507 zadan = 30% calego ruchu. # Ta sama blokada dziala na nikkel-art od 08.08. # claude Aga MB 2026-08-17 User-agent: GoogleOther Disallow: / User-agent: GoogleOther-Image Disallow: / User-agent: GoogleOther-Video Disallow: / # Nowsze nazwy asystentow AI — wpisane wprost, zeby polityka nie zalezala # od reguly ogolnej. claude Aga MB 2026-08-17 User-agent: OAI-SearchBot Allow: / User-agent: ClaudeBot Allow: / # PerplexityBot ZABLOKOWANY (decyzja Agi 2026-10-10: ~130 tys. zadan/dobe na foto4art.be i fotos4art.de). claude Aga MB User-agent: PerplexityBot Disallow: / User-agent: DuckAssistBot Allow: / User-agent: Google-Extended Allow: / # Applebot zablokowany CELOWO: Apple bardzo mocno crawlowal sklepy # i obciazal serwer — ta sama blokada dziala na nikkel-art. # Applebot-Extended nie crawluje (dotyczy tylko trenowania modeli), # blokujemy dla spojnosci. claude Aga MB 2026-08-17 User-agent: Applebot Disallow: / # ============================================ # SITEMAP # ============================================ Sitemap: https://www.foto4art.nl/sitemap-p-f4a-nl.xml Sitemap: https://www.foto4art.nl/sitemap_photos-parent.xml Sitemap: https://www.foto4art.nl/sitemap_cms.xml Sitemap: https://www.foto4art.nl/sitemap-upload.xml