From 155664bcce85e68a5ce333f90010a6fc063510bd Mon Sep 17 00:00:00 2001 From: Daniel Miessler Date: Wed, 23 Sep 2015 09:41:27 -0700 Subject: [PATCH] Added RobotsDisallowed content to Discovery/Web_Content --- .../Web_Content/Top10-RobotsDisallowed.txt | 10 + .../Web_Content/Top100-RobotsDisallowed.txt | 100 ++ .../Web_Content/Top1000-RobotsDisallowed.txt | 1000 +++++++++++++++++ .../Web_Content/Top500-RobotsDisallowed.txt | 500 +++++++++ 4 files changed, 1610 insertions(+) create mode 100644 Discovery/Web_Content/Top10-RobotsDisallowed.txt create mode 100644 Discovery/Web_Content/Top100-RobotsDisallowed.txt create mode 100644 Discovery/Web_Content/Top1000-RobotsDisallowed.txt create mode 100644 Discovery/Web_Content/Top500-RobotsDisallowed.txt diff --git a/Discovery/Web_Content/Top10-RobotsDisallowed.txt b/Discovery/Web_Content/Top10-RobotsDisallowed.txt new file mode 100644 index 0000000..907d259 --- /dev/null +++ b/Discovery/Web_Content/Top10-RobotsDisallowed.txt @@ -0,0 +1,10 @@ + +/ +/api/ +/admin/ +/wp-admin/ +/template/ +/data/ +/search/ +/install/ +/admin.php diff --git a/Discovery/Web_Content/Top100-RobotsDisallowed.txt b/Discovery/Web_Content/Top100-RobotsDisallowed.txt new file mode 100644 index 0000000..cb1ae55 --- /dev/null +++ b/Discovery/Web_Content/Top100-RobotsDisallowed.txt @@ -0,0 +1,100 @@ + +/ +/api/ +/admin/ +/wp-admin/ +/template/ +/data/ +/search/ +/install/ +/admin.php +/tupian +/js/ +/member.php +/includes/ +/uc_client/ +/userapp.php?mod=app&* +/uc_server/ +/tmp/ +/static/ +/source/ +/search.php +/modules/ +/misc.php +/home.php?mod=spacecp* +/forum.php?mod=redirect* +/forum.php?mod=post* +/connect.php +/config/ +/api.php +/*mobile=yes* +/*?mod=misc* +/*?mod=attachment* +/themes/ +/xmlrpc.php +/admin +/images/ +/templates/ +/plus/stow.php +/plus/search.php +/plus/rss.php +/plus/recommend.php +/plus/mytag_js.php +/plus/feedback_js.php +/plus/count.php +/cgi-bin/ +/user/ +/templets +/scripts/ +/plus/shops_buyaction.php +/plus/posttocar.php +/plus/erraddsave.php +/plus/disdls.php +/plus/carbuyaction.php +/plus/car.php +/plus/advancedsearch.php +/plus/ad_js.php +/member/ +/index.php +/include +/css/ +/ajax/ +/1000prosm.php +: +/wp-includes/ +/user +/search +/language/ +/install.php +/cgi-bin +/cache/ +/LICENSE.txt +/user/register/ +/user/password/ +/user/login/ +/update.php +/register/ +/profiles/ +/plugins/ +/page/ +/node/add/ +/misc/ +/error/ +/cron.php +/common/ +/comment/reply/ +/UPGRADE.txt +/MAINTAINERS.txt +/INSTALL.txt +/INSTALL.pgsql.txt +/INSTALL.mysql.txt +/CHANGELOG.txt +/*?* +/upload/ +/tag/ +/order/ +/mediamainlog.php +/logs/ +/lib/ +/inc/ +/fcmedianet.js diff --git a/Discovery/Web_Content/Top1000-RobotsDisallowed.txt b/Discovery/Web_Content/Top1000-RobotsDisallowed.txt new file mode 100644 index 0000000..734dac9 --- /dev/null +++ b/Discovery/Web_Content/Top1000-RobotsDisallowed.txt @@ -0,0 +1,1000 @@ + +/ +/api/ +/admin/ +/wp-admin/ +/template/ +/data/ +/search/ +/install/ +/admin.php +/tupian +/js/ +/member.php +/includes/ +/uc_client/ +/userapp.php?mod=app&* +/uc_server/ +/tmp/ +/static/ +/source/ +/search.php +/modules/ +/misc.php +/home.php?mod=spacecp* +/forum.php?mod=redirect* +/forum.php?mod=post* +/connect.php +/config/ +/api.php +/*mobile=yes* +/*?mod=misc* +/*?mod=attachment* +/themes/ +/xmlrpc.php +/admin +/images/ +/templates/ +/plus/stow.php +/plus/search.php +/plus/rss.php +/plus/recommend.php +/plus/mytag_js.php +/plus/feedback_js.php +/plus/count.php +/cgi-bin/ +/user/ +/templets +/scripts/ +/plus/shops_buyaction.php +/plus/posttocar.php +/plus/erraddsave.php +/plus/disdls.php +/plus/carbuyaction.php +/plus/car.php +/plus/advancedsearch.php +/plus/ad_js.php +/member/ +/index.php +/include +/css/ +/ajax/ +/1000prosm.php +: +/wp-includes/ +/user +/search +/language/ +/install.php +/cgi-bin +/cache/ +/LICENSE.txt +/user/register/ +/user/password/ +/user/login/ +/update.php +/register/ +/profiles/ +/plugins/ +/page/ +/node/add/ +/misc/ +/error/ +/cron.php +/common/ +/comment/reply/ +/UPGRADE.txt +/MAINTAINERS.txt +/INSTALL.txt +/INSTALL.pgsql.txt +/INSTALL.mysql.txt +/CHANGELOG.txt +/*?* +/upload/ +/tag/ +/order/ +/mediamainlog.php +/logs/ +/lib/ +/inc/ +/fcmedianet.js +/commons/ +/cmedianet +/cmdynet +/click.php +/category/ +/__media__/js/templates.js +/?q=user/register/ +/?q=user/password/ +/?q=user/login/ +/?q=search/ +/?q=node/add/ +/?q=comment/reply/ +/?q=admin/ +/*gclid +/*.swf +/ywidget +/wp-content/themes +/wp-content/plugins +/wp-admin +/wingBanner/ +/usr-bin +/udaup.php +/txt4.php +/txt3.php +/txt2.php +/txt1.php +/tns/ +/timetable +/super-product* +/static/*.html +/socs.js +/skin/ +/search_ds/ +/remittance/ +/redirect +/recipes/ +/rd2 +/rd1 +/primer-informers.php +/price_item.aspx +/poiskmainz2.php +/photoupload/ +/payment/ +/pay1000/in.php +/openapi/ +/obyavi.php +/new-product* +/new-product +/new-b-product* +/new-b-product +/manage/ +/login.php +/kabinet.php +/jsp/ +/informer.php +/info/rules +/index +/help.php? +/help.php +/goo2.php +/goo.php +/fonts/ +/firm/*/*.aspx +/estate/teaseritems +/estate/firmestate/firm +/download/ +/doski/sendpismo.php +/doski/senddoski.php +/doski/ +/doski.php/doski.php +/csagent/ +/community/ +/chatping/ +/catalog/spravochnaia-aptek.aspx +/catalog/search*.aspx +/catalog/raion.aspx +/catalog/drugstore.aspx +/catalog/default.aspx +/catalog/alert.aspx +/cart +/cabinet +/blank_paket2.php +/blank_paket.php +/bitrix/ +/banks/request +/b-product* +/b-product +/auth/ +/auth.php +/aff-bin +/advert.php +/abuse.php +/aboutinformer2.php +/aboutinformer +/5let.php +/2012.php +/1000text.php +/1000servis.php +/1000rassil.php +/1000rassil +/1000prosm +/1000perex.php +/1000paket +/1000p.php +/*register=yes +/*print_course=Y +/*logout=yes +/*login=yes +/*forgot_password=yes +/*change_password=yes +/*bitrix_*= +/*backurl=* +/*back_url_admin=* +/*back_url=* +/*auth=yes +/*BACK_URL=* +/*BACKURL=* +/*?print= +/*&print= +*search* +*products +*print +*mark_id* +*add* +*/trackback +/wp-register.php +/wp-includes +/wp-content/cache +/v/ +/user/logout/ +/upgrade/ +/tourdesc +/text +/temp/ +/tag +/system/ +/statistics.html +/services/ +/search* +/save +/plus/ +/personal/ +/order-option +/opinion +/note +/message.php +/media/ +/logout/ +/login/ +/libraries/ +/languages/ +/installation/ +/include/ +/img/ +/help/ +/final +/filter/tips/ +/engine/go.php +/engine/download.php +/detail/ +/delay +/counters +/contact/ +/components/ +/cert/ +/catalog/ +/cart/ +/board/ +/blocks/ +/basket/ +/apps/ +/administrator/ +/INSTALL.sqlite.txt +/?q=user/logout/ +/?q=filter/tips/ +/?level=12 +/? +/404.html +/*x= +/*utm_source=* +/*utm_medium=* +/*utm_content=* +/*utm_campaign=* +/*utm +/*tourpoisk +/*subaction=userinfo +/*sputprtn +/*search +/*print= +/*openstat +/*level=liger* +/*img/managers/docs/ +/*ggl +/*from=mail +/*do=stats +/*do=pm +/*do=lostpassword +/*do=lastcomments +/*do=feedback +/*do=addnews +/*act=viewinfo* +/*_openstat +/*WorkingVersion +/*Chld +/*?vidget +/*?mode=viewprofile +/*?from=begun +/*?fb_xd_fragment +/*? +/*/page= +/*.gif$ +espacepro* +developpeurs/ +?&what= +/z/ +/y/ +/xmlrpc/ +/xml/ +/xls_redirect/ +/wp-trackback +/wp-login.php +/wp-comments +/widget-118218/* +/webpage/ +/wap/ +/vodnik/tep/ +/vodnik/sekret/ +/vodnik/includes/ +/vodnik/htdocs/ +/vodnik/cgi-bin/ +/vklad/id +/video/ +/video-recepty +/users/ +/traf/map/ +/trackback +/top-sellers/ +/survey.php +/subcatalog/ +/store +/sms/ +/signup +/signaler-erreur/* +/signaler-erreur/ +/shop/ +/services +/sendurl +/sendcard +/send +/search.asp +/scripts/iefix.js +/scheme +/rules.html +/route/ +/respond.php +/require/ +/report.html +/register/vip_pay.html +/register.php +/region.php +/referers_list.php +/redo_form/ +/redirect/ +/receive.php +/realty_map_data/ +/quanto-vale-seu-apartamento/passo* +/pw_app.php +/pw_api.php +/pw_ajax.php +/profile.php +/profile +/product_by_id/ +/product/ +/pp/ +/post.php +/pm +/phpmailer/ +/pdd/user/stats/ +/pdd/themes/ +/pdd/random/ +/pdd/init/ +/pdd/exam/ +/pdd/change/ +/pda/ +/parking.php4 +/panier.html +/pages/displayCalculatorV2/ +/owners/ +/out/ +/out.php +/offers/buy/ +/notebooks/brand-packard%20bell/ +/notebooks/brand-/ +/network/ +/nalog?calc +/myauto +/mst/cache/images/ +/mode/ +/mobile/ +/members +/mail/ +/loyalty/ +/logout +/lang/ +/kurs/undefined +/kurs/news/ +/kurs/mforecast/ +/kurs/RSS/ +/kredit/id +/js_no_index/ +/job.php +/ipdata/ +/interest/ +/install +/ingredients/ +/infrastructure/get_geo/ +/index_dev.php +/index/ +/index.php/ +/index$ +/imprimer/* +/imprimer/ +/htdocs/ +/how_to_choose/95728/ +/hack/ +/groups +/goods_script.php +/go +/games/*/play +/game/*/play +/gallery.php +/gal/ +/forumcp.php +/forum/language/ +/forum/db/ +/forum/cgi-bin/ +/forum/cache/ +/forum/admin/ +/feeds/ +/feed.php +/examples/my-components/ +/examples/download/download_private/ +/engine/ajax/ +/eng +/email/ +/e-store/paid/detail.php +/e-store/affiliates/ +/dynamic +/dle-rules-page.html +/developpeurs/* +/deprecated_browser.html +/denunciar-anuncio/ +/db/autos/*?p +/db/autos.html?r +/db/ +/cycle_image.php +/customer/ +/cswjjd/ +/cssikexin/ +/createcard +/content1 +/content/links/my/ +/content/board/my/ +/content +/contacts/* +/complaint/ +/comparemod/*_vs$ +/comparemod/$ +/compare/*_vs$ +/communication/forum/user/ +/communication/forum/search/ +/communication/blog/search.php +/comments/feed/ +/comments +/comment.php +/collection/ +/cms/FR-FR/pneu-hiver/saison/hiver/VREDESTEIN/MICHELIN/X-ICE-XI3.html +/cms/FR-FR/pneu-hiver/saison/hiver/HANKOOK/VREDESTEIN/MICHELIN/ALPIN-5.html +/cms/FR-FR/pneu-4x4/pneu/4x4/TOYO/ATTURO/BRIDGESTONE/INSA-TURBO/BRIDGESTONE/ATTURO/AZ800.html +/cms/FR-FR/pneu-4x4/pneu/4x4/HANKOOK/HANKOOK/DYNAPRO-MT-RT03.html +/cms/FR-FR/pneu-4x4/pneu/4x4/BRIDGESTONE/BRIDGESTONE/TOYO/INSA-TURBO/HANKOOK/BF-GOODRICH/INSA-TURBO/RANGER.html +/cms/FR-FR/pneu-4x4/pneu/4x4/BF-GOODRICH/BRIDGESTONE/DUELER-A-T-694.html +/cms/FR-FR/pneu-4x4/pneu/4x4/ATTURO/BF-GOODRICH/ALL-TERRAIN-T-A-KO.html +/club/search/ +/club/messages/ +/club/log/ +/club/group/search/ +/club/gallery/tags/ +/club/forum/search/ +/club/$ +/ck.php +/checkout +/chatgatewaycounts/ +/catégorie/ +/catalog/product_compare/ +/cars/question/ +/captcha.php +/bundles/* +/blogs +/bin.aspx?ID* +/bbs/uc_server/ +/bbs/uc_client/ +/bbs/template/ +/bbs/static/ +/bbs/source/ +/bbs/search.php +/bbs/misc.php +/bbs/member.php +/bbs/install/ +/bbs/home.php?mod=spacecp* +/bbs/forum.php?mod=redirect* +/bbs/forum.php?mod=post* +/bbs/data/ +/bbs/config/ +/bbs/api/ +/bbs/api.php +/bbs/admin.php +/basket/add/* +/bank/upload.php +/author +/attachment/ +/asc/ +/archive/ +/appointment/ +/api +/annuaire-professio/ +/amy695806155/ +/ajax.php +/affiche.php +/admin2.php +/admin* +/addnews.html +/active.php +/about/actions/ +/ErrorLog/ +/?q=logout/ +/?q=contact/ +/?q=* +/?msk +/?favorites +/?* +/7294873 +/405.html +/13629693 +/*utm_source= +/*subpart=* +/*struct_id=* +/*register +/*price_min= +/*price_max= +/*pid= +/*orderby=* +/*order=* +/*name= +/*minprice=* +/*member* +/*maxprice=* +/*level=12* +/*index.php$ +/*index.php +/*image +/*from=adwords +/*filter_sended= +/*do=register +/*detail.php +/*ajax= +/*add_search=* +/*action=DELETE_FROM_COMPARE_LIST +/*action=BUY +/*action=ADD_TO_COMPARE_LIST +/*action=ADD2BASKET +/*_openstat= +/*Ya_ +/*SHOWALL_1= +/*QUERY= +/*Password +/*PAGE_NAME=user_post +/*PAGE_NAME=search +/*PAGE_NAME=detail_slide_show +/*?yclid +/*?st* +/*?pm* +/*?pdg +/*?brand_ +/*?adm +/*/slide_show/ +/*/search/ +/*/page-1/ +/*/gallery/*order=* +/*/?_openstat +/*.php +/*.pdf +/*.jpg$ +/*.aspx$ +/*.asp* +/*&st* +/*&pm* +*?page=1 +*?action +*? +*/print/ +*/from_ya +*/feed +*/comments +*/brand-/* +*/OID_* +*/?view=* +*/?sort=* +*/?lang.ru +*/?f=* +*.swf +*& +you +www2.123i.com.br +robots.txt +m.123i.com.br +ex: +arch.php +admin.php +/笠勝電子有限公司-找工作機會-68848493.htm +/~ +/zxjc/ +/zulin/admin/ +/zrlist.aspx +/zozm/ +/zoomify/ +/zhuanti/reiz/admin/ +/zhongda/admin/ +/zhiyucainuan/ +/zhicheng/admin/ +/zapomenute_prihlasovaci_udaje.html +/yzjj/ +/ypxsmg/ +/ypxrzg/ +/ypxomg/ +/ypxhmg/ +/yp/my/ +/yp/*.asp?* +/yp/*.asp$ +/youhui_n/ +/youhui/*.asp?* +/youhui/*.asp$ +/ylyat/ +/yjx001/ +/yesterday/ +/year/ +/xml_feeds/ +/xml +/xian/zhuanti +/xhzd/ +/xd_receiver.htm +/wuaiext/ +/wsauto/admin/ +/write_a_review.php +/wp-trackback.php +/wp-settings.php +/wp-rss2.php +/wp-rss.php +/wp-rdf.php +/wp-pass.php +/wp-mail.php +/wp-links-opml.php +/wp-feed.php +/wp-content/plugins/ +/wp-content/cache/ +/wp-content/ +/wp-config.php +/wp-commentsrss2.php +/wp-comments-post.php +/wp-blog-header.php +/wp-atom.php +/work1 +/wishlist/ +/wishlist +/wish/ +/wind/ +/wiki/ +/widget/head/login +/why/* +/week/ +/wed_n/ +/wed/*.asp?* +/wed/*.asp$ +/webservices/ +/weborder/ +/webgame/ +/webchatcce +/wb/ +/watch* +/wangyingran +/wagon/admin/ +/w/ +/vote/ +/vodnik/tmp/ +/vod-show-id-*-p-*.html$ +/vod-show*up*.html$ +/vod-show*hits*.html$ +/vod-search-* +/vod-play*.html$ +/vocabulary/ +/visitorregister* +/visitbelgrade.rs/ +/visitbelgrade.net/ +/vipcard/ +/vip/ +/vip +/views/ +/viewimg_* +/view.php +/vietnamese/-------------------------- +/videos/sponsor-story/ +/version-info/ +/velikosrce/ +/va/ +/v5/weborder/ +/v5/action/secweborder/ +/v5/PayCenter/ +/v3_specialnr/ +/v3_login/qzonelogin.html +/v3_index/wrong.html +/v1_popup/* +/v1_home/* +/uuseeimg/ +/users/popup/* +/users/headermenu +/users/ajax/* +/userauth/ +/user/. +/user/* +/user.php +/user.asp* +/upvoted/ +/uploads/ +/uploadfiles/ +/upload.php +/upimg/ +/upfile/* +/updates/ +/uncategorized/test/ +/ui/ +/ugc/ +/ucc +/uc +/ubeogradu.rs/ +/txt* +/tv_stanice_xml/ +/tupush/ +/tuan/*.asp?* +/tuan/*.asp$ +/travel/$ +/translator/ +/translate/ +/transfer/ +/tranergy +/trackback/ +/toutiaohuandeng* +/topusers/ +/topusers.php +/topten +/topic/*/smart/ +/top.asp +/tools07073 +/tools/ +/today/ +/to.htm?* +/thumb.php +/thread.php +/text/* +/tentarComprar/* +/temporada/*?r& +/temporada/*?q= +/tempcache/ +/tempImages/ +/tel-domain-registration/ +/tbxl/ +/sys +/syjc/ +/sydh/ +/sycon/ +/sycg2009/ +/survey/importCarownerData.aspx +/survey/SurveyFiles +/suggestion_keywords/autocomplete_suggestion +/submit/ +/submit.php +/story/sponsor-story/ +/story/draft/ +/story.php +/store_n/ +/store/*.asp?* +/store/*.asp$ +/stats/count +/stats/ +/station_ok/ +/static +/stars/ +/ssl +/ssi/ +/ssd/ +/src/ +/sponsors/ +/spider-trap +/sou-corretor-de-imoveis/ +/softeval.asp?* +/soft/previewimg.asp?softid=* +/soft/previewimg.asp +/soft/list.asp?classid=* +/soft/list.asp +/soft/download.asp?softid=* +/soft/download.asp +/so/ +/skins/ +/sjhd +/sitesearch.asp* +/sites/ +/simple/ +/signup* +/signin +/sign-out.html +/sign-in.html +/showpro.aspx +/showerr.asp +/shoppingcart.html +/shopping_cart +/shopping-item/* +/shop/shopmember/ +/shop/admin/ +/shitou/ +/shimisi/ +/shengyuan/admin/ +/share/ +/shangjiahudong +/sezioni/976/test-approvazione +/sezioni/949/condizioni-generali-dei-servizi-offerti +/sezioni/1423/news-estate +/sezioni/1369/palinsesto-24-30-agosto +/sezioni/1367/palinsesto-17-23-agosto +/sezioni/1366/palinsesto-10-16-agosto +/sezioni/1363/palinsesto-3-9-agosto +/sezioni/1352/html5-webradio +/sezioni/1343/mobile-layout-test +/sezioni/1320/primo-piano-app-105 +/sezioni/1306/ylenia +/sezioni/1292/il-blog-di-leone-dilernia +/sezioni/1269/widget-news +/sezioni/1051/prova-diretta +/sezioni/1006/test +/servis-za-prodavce/ +/services/* +/servicequery +/serverList/ +/server/ +/senyang/ +/sendpwd.php +/seminarprojects/tag/ +/sell/*?* +/selection_station_de_montage/ +/securimage/ +/secure/ +/seasons_commu/* +/searchurl/ +/searchurl.php +/search_keywords/* +/search_* +/searchProduct.ac +/search.asp?word=* +/school-lunch/* +/save_data.asp +/save-pro* +/save-biz* +/sales/ +/sales +/sakura/ +/s/tao/ +/rss/ +/rss.xml +/rss-feeds/ +/rongji/admin/ +/robot/ +/rjs/ +/reviews-Ratings +/reviews +/result/ +/result.php +/resource/activity/buyingingroup/index.html +/res/ +/related/* +/rel/q/ +/register.html +/regAndAsk.html* +/recommended/ +/recherche +/recent/ +/readme.html +/read.php +/rd/ +/rd.php* +/random +/qy_xx.aspx +/qy_pro.aspx +/qy_msg.aspx +/qy_about.aspx +/quisDizer/ +/quero-contribuir-com-informacoes/* +/quero-comercializar-meu-apartamento/* +/qqlive.html +/qqlive.htm +/qqajax.php +/qiushengzhilu* +/qiche/*.asp?* +/qiche/*.asp$ +/pwr/simplyink/pwr/6n66vqj2/rawdata/ +/pwr/simplyink/pwr/6n66vqj2/debug.html +/pwr/4inkjets/pwr/n9kd9e5d/rawdata/ +/pwr/4inkjets/pwr/n9kd9e5d/debug.html +/pwr/123inkjets/pwr/885jwpi5/rawdata/ +/pwr/123inkjets/pwr/885jwpi5/debug.html +/pw8/ +/publish/ +/public_html/ +/psreports/ +/psp/ +/psc/ +/profile/ +/produits/ +/produits-de-bar/contact_us.php +/products/ +/productreview/ +/product/solutions/solutions/legal_notice/about/index.html +/product/service/service/legal_notice/about/about_us.html +/product/list.aspx +/product/legal_notice/product/supervision/company_news/index.html +/product/legal_notice/cooperation/solutions/about/supervision/product/mall.html +/product/legal_notice/cooperation/legal_notice/solutions/company_news/supervision/joinus.html +/product/legal_notice/cooperation/legal_notice/solutions/company_news/20140412v2.html +/product/legal_notice/cooperation/company_news/company_news/index.html +/product/index.aspx +/product/cooperation/company_news/service/legal_notice/index.html +/product/company_news/supervision/company_news/cooperation/policy.html +/product/company_news/service/company_news/solutions/index.html +/product/company_news/legal_notice/supervision/legal_notice/index.html +/product/company_news/legal_notice/cooperation/service/index.html +/product/company_news/cooperation/company_news/solutions/index.html +/product/company_news/about/legal_notice/supervision/index.html +/product/advertisement.html +/product.ac +/pro/*/reserva* +/pro/*/rating/* +/pro/*/ldr* +/pro/*/date* +/pro/*/contents* +/pro/*/contacts* +/pro/*/bizmedia* +/pro/*/biz* +/private +/pri_submit.asp +/post1/ +/post/*?* +/post/*.asp?* +/post/*.asp$ +/pomoc-a-kontakt/kontakt/ +/poll/ +/poisk/ +/pma/ +/plus/list.php?tid=2078&TotalResult=410&PageNo= +/plus/list.php?tid=2078 +/plugin.php?id=* +/player/* +/pingrong/ +/picture-gallery/sponsor-story/ +/pic/ +/pianor/ +/phpsso_server +/phpcms +/phonebook/*.asp?* +/phonebook/*.asp$ +/phb/jfb.htm* +/pharmacies/ +/perto-de-mim/ +/personal_ads/ +/permalink/ +/peradmin/* +/pdf/ +/pdf.asp +/pay/ +/parapharmacie/pharmacies/ +/panier +/pages/interactives/sponsor-story/ +/page_confirm.asp +/pageRight.html* +/page.php +/p.html +/organiserenquiry +/order +/optsale/last/toshiba_u500_1f4.html +/optsale/last/toshiba_u500_1dq.html +/optsale/last/toshiba_u500_18p.html +/optsale/last/toshiba_t130_16u.html +/optsale/last/toshiba_t110_12g.html +/optsale/last/toshiba_qosmio_x500_110.html diff --git a/Discovery/Web_Content/Top500-RobotsDisallowed.txt b/Discovery/Web_Content/Top500-RobotsDisallowed.txt new file mode 100644 index 0000000..89eaf46 --- /dev/null +++ b/Discovery/Web_Content/Top500-RobotsDisallowed.txt @@ -0,0 +1,500 @@ + +/ +/api/ +/admin/ +/wp-admin/ +/template/ +/data/ +/search/ +/install/ +/admin.php +/tupian +/js/ +/member.php +/includes/ +/uc_client/ +/userapp.php?mod=app&* +/uc_server/ +/tmp/ +/static/ +/source/ +/search.php +/modules/ +/misc.php +/home.php?mod=spacecp* +/forum.php?mod=redirect* +/forum.php?mod=post* +/connect.php +/config/ +/api.php +/*mobile=yes* +/*?mod=misc* +/*?mod=attachment* +/themes/ +/xmlrpc.php +/admin +/images/ +/templates/ +/plus/stow.php +/plus/search.php +/plus/rss.php +/plus/recommend.php +/plus/mytag_js.php +/plus/feedback_js.php +/plus/count.php +/cgi-bin/ +/user/ +/templets +/scripts/ +/plus/shops_buyaction.php +/plus/posttocar.php +/plus/erraddsave.php +/plus/disdls.php +/plus/carbuyaction.php +/plus/car.php +/plus/advancedsearch.php +/plus/ad_js.php +/member/ +/index.php +/include +/css/ +/ajax/ +/1000prosm.php +: +/wp-includes/ +/user +/search +/language/ +/install.php +/cgi-bin +/cache/ +/LICENSE.txt +/user/register/ +/user/password/ +/user/login/ +/update.php +/register/ +/profiles/ +/plugins/ +/page/ +/node/add/ +/misc/ +/error/ +/cron.php +/common/ +/comment/reply/ +/UPGRADE.txt +/MAINTAINERS.txt +/INSTALL.txt +/INSTALL.pgsql.txt +/INSTALL.mysql.txt +/CHANGELOG.txt +/*?* +/upload/ +/tag/ +/order/ +/mediamainlog.php +/logs/ +/lib/ +/inc/ +/fcmedianet.js +/commons/ +/cmedianet +/cmdynet +/click.php +/category/ +/__media__/js/templates.js +/?q=user/register/ +/?q=user/password/ +/?q=user/login/ +/?q=search/ +/?q=node/add/ +/?q=comment/reply/ +/?q=admin/ +/*gclid +/*.swf +/ywidget +/wp-content/themes +/wp-content/plugins +/wp-admin +/wingBanner/ +/usr-bin +/udaup.php +/txt4.php +/txt3.php +/txt2.php +/txt1.php +/tns/ +/timetable +/super-product* +/static/*.html +/socs.js +/skin/ +/search_ds/ +/remittance/ +/redirect +/recipes/ +/rd2 +/rd1 +/primer-informers.php +/price_item.aspx +/poiskmainz2.php +/photoupload/ +/payment/ +/pay1000/in.php +/openapi/ +/obyavi.php +/new-product* +/new-product +/new-b-product* +/new-b-product +/manage/ +/login.php +/kabinet.php +/jsp/ +/informer.php +/info/rules +/index +/help.php? +/help.php +/goo2.php +/goo.php +/fonts/ +/firm/*/*.aspx +/estate/teaseritems +/estate/firmestate/firm +/download/ +/doski/sendpismo.php +/doski/senddoski.php +/doski/ +/doski.php/doski.php +/csagent/ +/community/ +/chatping/ +/catalog/spravochnaia-aptek.aspx +/catalog/search*.aspx +/catalog/raion.aspx +/catalog/drugstore.aspx +/catalog/default.aspx +/catalog/alert.aspx +/cart +/cabinet +/blank_paket2.php +/blank_paket.php +/bitrix/ +/banks/request +/b-product* +/b-product +/auth/ +/auth.php +/aff-bin +/advert.php +/abuse.php +/aboutinformer2.php +/aboutinformer +/5let.php +/2012.php +/1000text.php +/1000servis.php +/1000rassil.php +/1000rassil +/1000prosm +/1000perex.php +/1000paket +/1000p.php +/*register=yes +/*print_course=Y +/*logout=yes +/*login=yes +/*forgot_password=yes +/*change_password=yes +/*bitrix_*= +/*backurl=* +/*back_url_admin=* +/*back_url=* +/*auth=yes +/*BACK_URL=* +/*BACKURL=* +/*?print= +/*&print= +*search* +*products +*print +*mark_id* +*add* +*/trackback +/wp-register.php +/wp-includes +/wp-content/cache +/v/ +/user/logout/ +/upgrade/ +/tourdesc +/text +/temp/ +/tag +/system/ +/statistics.html +/services/ +/search* +/save +/plus/ +/personal/ +/order-option +/opinion +/note +/message.php +/media/ +/logout/ +/login/ +/libraries/ +/languages/ +/installation/ +/include/ +/img/ +/help/ +/final +/filter/tips/ +/engine/go.php +/engine/download.php +/detail/ +/delay +/counters +/contact/ +/components/ +/cert/ +/catalog/ +/cart/ +/board/ +/blocks/ +/basket/ +/apps/ +/administrator/ +/INSTALL.sqlite.txt +/?q=user/logout/ +/?q=filter/tips/ +/?level=12 +/? +/404.html +/*x= +/*utm_source=* +/*utm_medium=* +/*utm_content=* +/*utm_campaign=* +/*utm +/*tourpoisk +/*subaction=userinfo +/*sputprtn +/*search +/*print= +/*openstat +/*level=liger* +/*img/managers/docs/ +/*ggl +/*from=mail +/*do=stats +/*do=pm +/*do=lostpassword +/*do=lastcomments +/*do=feedback +/*do=addnews +/*act=viewinfo* +/*_openstat +/*WorkingVersion +/*Chld +/*?vidget +/*?mode=viewprofile +/*?from=begun +/*?fb_xd_fragment +/*? +/*/page= +/*.gif$ +espacepro* +developpeurs/ +?&what= +/z/ +/y/ +/xmlrpc/ +/xml/ +/xls_redirect/ +/wp-trackback +/wp-login.php +/wp-comments +/widget-118218/* +/webpage/ +/wap/ +/vodnik/tep/ +/vodnik/sekret/ +/vodnik/includes/ +/vodnik/htdocs/ +/vodnik/cgi-bin/ +/vklad/id +/video/ +/video-recepty +/users/ +/traf/map/ +/trackback +/top-sellers/ +/survey.php +/subcatalog/ +/store +/sms/ +/signup +/signaler-erreur/* +/signaler-erreur/ +/shop/ +/services +/sendurl +/sendcard +/send +/search.asp +/scripts/iefix.js +/scheme +/rules.html +/route/ +/respond.php +/require/ +/report.html +/register/vip_pay.html +/register.php +/region.php +/referers_list.php +/redo_form/ +/redirect/ +/receive.php +/realty_map_data/ +/quanto-vale-seu-apartamento/passo* +/pw_app.php +/pw_api.php +/pw_ajax.php +/profile.php +/profile +/product_by_id/ +/product/ +/pp/ +/post.php +/pm +/phpmailer/ +/pdd/user/stats/ +/pdd/themes/ +/pdd/random/ +/pdd/init/ +/pdd/exam/ +/pdd/change/ +/pda/ +/parking.php4 +/panier.html +/pages/displayCalculatorV2/ +/owners/ +/out/ +/out.php +/offers/buy/ +/notebooks/brand-packard%20bell/ +/notebooks/brand-/ +/network/ +/nalog?calc +/myauto +/mst/cache/images/ +/mode/ +/mobile/ +/members +/mail/ +/loyalty/ +/logout +/lang/ +/kurs/undefined +/kurs/news/ +/kurs/mforecast/ +/kurs/RSS/ +/kredit/id +/js_no_index/ +/job.php +/ipdata/ +/interest/ +/install +/ingredients/ +/infrastructure/get_geo/ +/index_dev.php +/index/ +/index.php/ +/index$ +/imprimer/* +/imprimer/ +/htdocs/ +/how_to_choose/95728/ +/hack/ +/groups +/goods_script.php +/go +/games/*/play +/game/*/play +/gallery.php +/gal/ +/forumcp.php +/forum/language/ +/forum/db/ +/forum/cgi-bin/ +/forum/cache/ +/forum/admin/ +/feeds/ +/feed.php +/examples/my-components/ +/examples/download/download_private/ +/engine/ajax/ +/eng +/email/ +/e-store/paid/detail.php +/e-store/affiliates/ +/dynamic +/dle-rules-page.html +/developpeurs/* +/deprecated_browser.html +/denunciar-anuncio/ +/db/autos/*?p +/db/autos.html?r +/db/ +/cycle_image.php +/customer/ +/cswjjd/ +/cssikexin/ +/createcard +/content1 +/content/links/my/ +/content/board/my/ +/content +/contacts/* +/complaint/ +/comparemod/*_vs$ +/comparemod/$ +/compare/*_vs$ +/communication/forum/user/ +/communication/forum/search/ +/communication/blog/search.php +/comments/feed/ +/comments +/comment.php +/collection/ +/cms/FR-FR/pneu-hiver/saison/hiver/VREDESTEIN/MICHELIN/X-ICE-XI3.html +/cms/FR-FR/pneu-hiver/saison/hiver/HANKOOK/VREDESTEIN/MICHELIN/ALPIN-5.html +/cms/FR-FR/pneu-4x4/pneu/4x4/TOYO/ATTURO/BRIDGESTONE/INSA-TURBO/BRIDGESTONE/ATTURO/AZ800.html +/cms/FR-FR/pneu-4x4/pneu/4x4/HANKOOK/HANKOOK/DYNAPRO-MT-RT03.html +/cms/FR-FR/pneu-4x4/pneu/4x4/BRIDGESTONE/BRIDGESTONE/TOYO/INSA-TURBO/HANKOOK/BF-GOODRICH/INSA-TURBO/RANGER.html +/cms/FR-FR/pneu-4x4/pneu/4x4/BF-GOODRICH/BRIDGESTONE/DUELER-A-T-694.html +/cms/FR-FR/pneu-4x4/pneu/4x4/ATTURO/BF-GOODRICH/ALL-TERRAIN-T-A-KO.html +/club/search/ +/club/messages/ +/club/log/ +/club/group/search/ +/club/gallery/tags/ +/club/forum/search/ +/club/$ +/ck.php +/checkout +/chatgatewaycounts/ +/catégorie/ +/catalog/product_compare/ +/cars/question/ +/captcha.php +/bundles/* +/blogs +/bin.aspx?ID*