JAVA网络爬虫批量下载豌豆夹中的应用

            package
             studyjava;


            import
             java.io.BufferedReader;

            import
             java.io.File;

            import
             java.io.FileOutputStream;

            import
             java.io.FileWriter;

            import
             java.io.IOException;

            import
             java.io.InputStream;

            import
             java.io.InputStreamReader;

            import
             java.net.HttpURLConnection;

            import
             java.net.MalformedURLException;

            import
             java.net.URL;

            import
             java.net.URLConnection;

            import
             java.util.regex.Matcher;

            import
             java.util.regex.Pattern;

            import
             java.util.stream.Stream;


            import
             java.net.HttpURLConnection;

            import
             javax.xml.transform.stream.StreamSource;



            public
            class
             client{
    
            public
            static
            void main(String[] args) throws IOException
    {   
        //在D盘创建两份文件，一份为获取到的所有APK下载链接，一份为下载的所有应用名称
        File f1=new File("D:\\down.txt");
        File f2=new File("D:\\down_appname.txt");
        if(f1.exists())
            f1.delete();
        else
            f1.createNewFile();
        
        if(f2.exists())
            f2.delete();
        else
            f2.createNewFile();
        
        FileWriter fw=new FileWriter(f1,true);
        FileWriter fw2=new FileWriter(f2,true);
        
        String str;
        //下载的网址，如豌豆夹必备应用页
        URL url=new URL("http://www.wandoujia.com/essential/app");
        //下面就是给定一个网址，你怎么去获取它的源码咯
        URLConnection conn=url.openConnection();
        BufferedReader br=new BufferedReader(new InputStreamReader(conn.getInputStream()));
        //下面是争对每一行输入流进行过滤，是网址的给我弄出来，然后再创建下载链接while((str=br.readLine())!=null)
        {   
            Pattern pattern=Pattern.compile("http://apps.wandoujia.com/apps/.+.([a-z]+\\.)*.+/download\"\\sdownload=\".+\\s*.*\\.apk\"");
            Matcher matcher=pattern.matcher(str);
            if(matcher.find())
            {   //正式写入下载网址和应用名称到相应文件中
                String downurl=matcher.group().split("\\s")[0].split("\"")[0];
                String downname=matcher.group().split("=")[1];
                fw.write(downurl);
                fw.write("\n");
                fw2.write(downname);
                fw2.write("\n");
                //下面是下载应用了byte[] buffer = newbyte[4 * 1024];
                //获取应用的网络资源连接，
                URL urldown=new URL(downurl);
                HttpURLConnection conn2=(HttpURLConnection)urldown.openConnection();
                conn2.setDoInput(true);
                conn2.connect();
                //获取输入流
                InputStream in=conn2.getInputStream();
                //创建放置下载应用的文件夹
                File dir=new File("D:\\downapp");
                if(!dir.exists())
                    dir.mkdir();
                //创建下载的应用，文件名和存放路径
                File appdown=new File(dir,downname.split("\"")[1]);
                if(!appdown.exists())
                    appdown.createNewFile();
                //获取输出流
                FileOutputStream out=new FileOutputStream(appdown);
                
                int br1=-1;
                while((br1=(in.read(buffer)))!=-1)
                    {
                    out.write(buffer, 0, br1);                
                    }
                out.flush();
                in.close();
                out.close();

            }
        }
        fw.close();
        fw2.close();
}
}