* | - /wp-admin/
- /go*/
- /go/*
- /go/
- /wp-json/
- /wp-json*/
- /wp-json/*
- /readme.html
- /license.txt
- /xmlrpc.php
- /404
- /div
- /404.html
- /wp-login.php
- /wp-login.php*
- /*.cur
- /grabbing.cur
- /?attachment_id=*
- /?attachment_id
|
MJ12bot | - All subpages are denied to crawl
|
AhrefsBot | - All subpages are denied to crawl
|
SeznamBot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
TwengaBot-2.0 | - All subpages are denied to crawl
|
BotRightHere | - All subpages are denied to crawl
|
WebZip | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
larbin | - All subpages are denied to crawl
|
b2w/0.1 | - All subpages are denied to crawl
|
Copernic | - All subpages are denied to crawl
|
psbot | - All subpages are denied to crawl
|
Python-urllib | - All subpages are denied to crawl
|
URL_Spider_Pro | - All subpages are denied to crawl
|
CherryPicker | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
EmailCollector | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
EmailSiphon | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebBandit | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
EmailWolf | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
ExtractorPro | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
CopyRightCheck | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Crescent | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
SiteSnagger | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
ProWebWalker | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
CheeseBot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
LNSpiderguy | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Alexibot | - All subpages are denied to crawl
|
Teleport | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
TeleportPro | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
MIIxpc | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Telesoft | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Website Quester | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
moget/2.1 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebZip/4.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebStripper | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebSauger | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebCopier | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
NetAnts | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Mister PiX | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebAuto | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
TheNomad | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WWW-Collector-E | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
RMA | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
libWeb/clsHTTP | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
asterias | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
httplib | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
turingos | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
spanner | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
InfoNaviRobot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Harvest/1.5 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Bullseye/1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; BullsEye; Windows 95) | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Crescent Internet ToolPak HTTP OLE Control v.1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
CherryPickerSE/1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
CherryPickerElite/1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebBandit/3.50 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
NICErsPRO | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Microsoft URL Control - 5.01.4511 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
DittoSpyder | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Foobot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
SpankBot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
BotALot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
lwp-trivial/1.34 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
lwp-trivial | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
BunnySlippers | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Microsoft URL Control - 6.00.8169 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
URLy Warning | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Wget/1.6 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Wget/1.5.3 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Wget | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
LinkWalker | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
cosmos | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
moget | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
hloader | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
humanlinks | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
LinkextractorPro | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Offline Explorer | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Mata Hari | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
LexiBot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Web Image Collector | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
The Intraformant | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
True_Robot/1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
True_Robot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
BlowFish/1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
JennyBot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
MIIxpc/4.2 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
BuiltBotTough | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
ProPowerBot/2.14 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
BackDoorBot/1.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
toCrawl/UrlDispatcher | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebEnhancer | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
suzuran | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
TightTwatBot | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
VCI WebViewer VCI WebViewer Win32 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
VCI | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Szukacz/1.4 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
QueryN Metasearch | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Openfind data gatherer | - All subpages are denied to crawl
|
Openfind | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Zeus | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
RepoMonkey Bait & Tackle/v1.01 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
RepoMonkey | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Microsoft URL Control | - All subpages are denied to crawl
|
Openbot | - All subpages are denied to crawl
|
URL Control | - All subpages are denied to crawl
|
Zeus Link Scout | - All subpages are denied to crawl
|
Zeus 32297 Webster Pro V2.9 Win32 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Webster Pro | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
EroCrawler | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
LinkScan/8.1a Unix | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Keyword Density/0.9 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Kenjin Spider | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
Iron33/1.0.2 | - All subpages are denied to crawl
|
Bookmark search tool | - All subpages are denied to crawl
|
GetRight/4.2 | - All subpages are denied to crawl
|
FairAd Client | - All subpages are denied to crawl
|
Gaisbot | - All subpages are denied to crawl
|
Aqua_Products | - All subpages are denied to crawl
|
Radiation Retriever 1.1 | - All subpages are denied to crawl
|
Flaming AttackBot | - All subpages are denied to crawl
|
Oracle Ultra Search | - All subpages are denied to crawl
|
MSIECrawler | - All subpages are denied to crawl
|
PerMan | - All subpages are denied to crawl
|
searchpreview | - All subpages are denied to crawl
|
TurnitinBot | - All subpages are denied to crawl
|
wget | - All subpages are denied to crawl
|
WebZIP/4.21 | - All subpages are denied to crawl
|
WebZIP/5.0 | - All subpages are denied to crawl
|
HTTrack 3.0 | - All subpages are denied to crawl
|
TurnitinBot/1.5 | - All subpages are denied to crawl
|
WebCopier v3.2a | - All subpages are denied to crawl
|
WebCapture 2.0 | - All subpages are denied to crawl
- All subpages are allowed to crawl
|
WebCopier v.2.2 | - All subpages are denied to crawl
|
baiduspider | - All subpages are denied to crawl
|
BaiDuSpider | - All subpages are denied to crawl
|
Yandex | - All subpages are denied to crawl
|
A6-Indexer | - All subpages are allowed to crawl
|
Aboundex | - All subpages are allowed to crawl
|
BacklinkCrawler | - All subpages are allowed to crawl
|
Black Hole | - All subpages are allowed to crawl
|
ca-crawler | - All subpages are allowed to crawl
|
CCBot | - All subpages are allowed to crawl
|
CCBot/2.0 | - All subpages are allowed to crawl
|
Cegbfeieh | - All subpages are allowed to crawl
|
DBot | - All subpages are allowed to crawl
|
EasouSpider | - All subpages are allowed to crawl
|
ECCP | - All subpages are allowed to crawl
|
Exabot/3.0 | - All subpages are allowed to crawl
|
fr-crawler | - All subpages are allowed to crawl
|
Friendly Crawler | - All subpages are allowed to crawl
|
GigablastOpenSource | - All subpages are allowed to crawl
|
Goodzer/2.0 | - All subpages are allowed to crawl
|
GrapeshotCrawler/2.0 | - All subpages are allowed to crawl
|
heritrix/1.14.4 | - All subpages are allowed to crawl
|
HubSpot Crawler 1.0 | - All subpages are allowed to crawl
|
IstellaBot | - All subpages are allowed to crawl
|
Konqueror/3.5 | - All subpages are allowed to crawl
|
Lipperhey SEO Service | - All subpages are allowed to crawl
|
meanpathbot | - All subpages are allowed to crawl
|
MixrankBot | - All subpages are allowed to crawl
|
mozilla/4 | - All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; MSIE 4.0; Windows 95) | - All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; MSIE 4.0; Windows 98) | - All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; MSIE 4.0; Windows NT) | - All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; MSIE 4.0; Windows XP) | - All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000) | - All subpages are allowed to crawl
|
Mozilla/4.0 (compatible; MSIE 4.0; Windows ME) | - All subpages are allowed to crawl
|
mozilla/5 | - All subpages are allowed to crawl
|
nbot/2.0 | - All subpages are allowed to crawl
|
NetzCheckBot/1.0 | - All subpages are allowed to crawl
|
oBot/2.3.1 | - All subpages are allowed to crawl
|
Openfind data gathere | - All subpages are allowed to crawl
|
PagesInventory | - All subpages are allowed to crawl
|
panscient.com | - All subpages are allowed to crawl
|
Riddler | - All subpages are allowed to crawl
|
RU_Bot | - All subpages are allowed to crawl
|
ScreenerBot Crawler Beta 2.0 | - All subpages are allowed to crawl
|
SemrushBot/0.98~bl | - All subpages are allowed to crawl
|
SEOENGWorldBot | - All subpages are allowed to crawl
|
SEOkicks-Robot | - All subpages are allowed to crawl
|
seplinkbot | - All subpages are allowed to crawl
|
seplinkbot/1.0 | - All subpages are allowed to crawl
|
Sistrix | - All subpages are allowed to crawl
|
SMTBot/1.0 | - All subpages are allowed to crawl
|
spbot | - All subpages are allowed to crawl
|
Sogou web spider | - All subpages are allowed to crawl
|
Titan | - All subpages are allowed to crawl
|
URLAppendBot | - All subpages are allowed to crawl
|
voltron | - All subpages are allowed to crawl
|
WBSearchBot | - All subpages are allowed to crawl
|
webcheck 1.10.4 | - All subpages are allowed to crawl
|
WebmasterWorldForumBot | - All subpages are allowed to crawl
|
WeSEE | - All subpages are allowed to crawl
|
WorldBrewBot/2.1 | - All subpages are allowed to crawl
|
www.integromedb.org/Crawler | - All subpages are allowed to crawl
|
Xenu Link Sleuth | - All subpages are allowed to crawl
|
Xenu's Link Sleuth 1.1c | - All subpages are allowed to crawl
|
XoviBot | - All subpages are allowed to crawl
|
ZumBot | - All subpages are allowed to crawl
|
Baiduspider | - All subpages are denied to crawl
|
Baiduspider+ | - All subpages are denied to crawl
|
sch-fast-se-crawl02.osl.basefarm.net | - All subpages are denied to crawl
|
sch-fast-se-crawl04.osl.basefarm.net | - All subpages are denied to crawl
|
ichiro | - All subpages are denied to crawl
|
NaverBot | - All subpages are denied to crawl
|
Yeti | - All subpages are denied to crawl
|
Baiduspider-video | - All subpages are denied to crawl
|
Baiduspider-image | - All subpages are denied to crawl
|
sogou spider | - All subpages are denied to crawl
|
YoudaoBot | - All subpages are denied to crawl
|
007ac9 | - All subpages are denied to crawl
|