2014-12-25 • fofa google search shodan url 搜索引擎

利用搜索引擎批量抓取url

有的时候爆出0day，我们需要抓取大量的url进行测试，这个时候需要用到搜索引擎，常用的有：google，fofa，shodan

下面是整理的利用js对对不同搜索引擎进行批量抓取的代码：

fofa: （如何使用：StartReq(搜索语法,开始页码,结束页码) ）

StartReq('body=wooyun',1,10)
function StartReq(q,startpage,endpage){
for(var i=startpage;i<=endpage;i++){
Req(i+"q="+encodeURIComponent(q)+"&qbase64="+btoa(q));
}
}
function Connection(Sendtype,url,content,callback){
if (window.XMLHttpRequest){
var xmlhttp=new XMLHttpRequest();
}
else{
var xmlhttp=new ActiveXObject("Microsoft.XMLHTTP");
}
 xmlhttp.onreadystatechange=function(){
if(xmlhttp.readyState==4&&xmlhttp.status==200)
{
 callback(xmlhttp.responseText);
}
}
 xmlhttp.open(Sendtype,url,true);
 xmlhttp.setRequestHeader("Content-Type","application/x-www-form-urlencoded");
 xmlhttp.send(content);
}
function Req(searchString){
var searchurl = "http://fofa.so/search/result?page="+searchString;
Connection("GET",searchurl,"",function(callback){
var result = $(callback);
 result.find('div.col-lg-4 a').each(function(i,o){
var o = $(o);
if(o.attr('target')=="_blank"){
if(o.attr('href').indexOf('/search/checkapp?all=true&host=')){
 console.log(o.attr('href'));
}
}
})
})
}

google:

StartReq("site:xss1.com",1,1);
var tmp = [];
var HerfRegExp = /http:\/\/\w.*\/|https:\/\/\w.*\//;
document.body.appendChild(document.createElement('script')).src='//code.jquery.com/jquery-1.9.1.min.js';
function StartReq(q,startpage,endpage){
for(var i=startpage;i<=endpage;i++){
if(i==1){
Req("q="+encodeURIComponent(q)+"&start=100&num=100&newwindow="+i);
}
else{
Req("q="+encodeURIComponent(q)+"&start="+(i*100)+"&num=100&newwindow="+i);
}
}
}
function Connection(Sendtype,url,content,callback){
if (window.XMLHttpRequest){
var xmlhttp=new XMLHttpRequest();
}
else{
var xmlhttp=new ActiveXObject("Microsoft.XMLHTTP");
}
 xmlhttp.onreadystatechange=function(){
if(xmlhttp.readyState==4&&xmlhttp.status==200)
{
 callback(xmlhttp.responseText);
}
}
 xmlhttp.open(Sendtype,url,true);
 xmlhttp.setRequestHeader("Content-Type","application/x-www-form-urlencoded");
 xmlhttp.send(content);
}
function Req(searchString){
var searchurl = "https://www.google.com.hk/search?"+searchString;
Connection("GET",searchurl,"",function(callback){
var result = $(callback);
 result.find('div.rc h3.r a').each(function(i,o){
var o = $(o);
 tmp.push(String(HerfRegExp.exec(o.attr('href'))));
})
})
}

最后的结果不会输出，会存入到tmp 数组，方便去重，如果需要输出可以自行加个循环tmp 把值打印出来

如何使用：StartReq(搜索语法,开始页码,结束页码)

shodan：

var url = "http://www.shodanhq.com/search?q=关键字&page=";
for(var i=1;i<101;i++){
var request = null;
if (window.ActiveXObject) {
 request = new ActiveXObject("Microsoft.XMLHTTP");
}else {
 request = new XMLHttpRequest();
}
 request.open("GET",url+i, false);
 request.setRequestHeader('If-Modified-Since', '0');
 request.send(null);
var str = request.responseText;
 str = str.replace(/\r/g,"");
 str = str.replace(/\n/g,"");
var urls = [];
 str.replace(/\<div class=\'ip\'>.*?<a href=\".*?\">(.*?)<\/a>.*?<\/div>/ig, function(a,b) {
 urls.push(b);
});
 console.info(urls.join('\n'));
}

新版的shodan：

var url = "https://www.shodan.io/search?query=port%3A27017&page=";
for(var i=1;i<101;i++){
var request = null;
if (window.ActiveXObject) {
 request = new ActiveXObject("Microsoft.XMLHTTP");
}else {
 request = new XMLHttpRequest();
}
 request.open("GET",url+i, false);
 request.setRequestHeader('If-Modified-Since', '0');
 request.send(null);
var str = request.responseText;
 str = str.replace(/\r/g,"");
 str = str.replace(/\n/g,"");
var urls = [];
 str.replace(/\<div class=\"ip\">.*?<a href=\".*?\">(.*?)<\/a>.*?<\/div>/ig, function(a,b) {
 urls.push(b);
});
 console.info(urls.join('\n'));
}

以上内容整理自：http://zone.wooyun.org/content/16840

kow