利用HtmlUtil 从网页提取数据

最近炒股，发现几个好的网站（散户大家庭http://www.shdjt.com/  股票大单网http://www.gupiaodadan.cn/），提供了每天大量分析数据------入主力持仓排名、大单买入排名、板块资金流入排名、DDX排名等等，于是我利用HtmlUtil 工具包，写了一个程序，把这些数据从网页上提取出来，保存在我自己的Oracle数据库里面，以便进行统计分析，从中找出规律，呵呵。。。。希望大发一笔，下面是具体的程序。

 

 

 

 

 【开发环境】MyEclipse6.5.1、Oracle9i

 

【所需要的库文件】

 htmlunit-2.5.zip                                       下载地址 http://htmlunit.sourceforge.net/

 Oracle9i JDBC驱动程序classes12.jar          在Oracle9i的安装目录下面的JDBC文件夹里有

 

  【数据库设计】

--股票目录
drop table gpml;
create table gpml(
   gpdm   varchar2(6) primary key,--股票代码
   gpmc   varchar2(20),--股票名称
   ltgs   number(10),--流通股数
   zgs    number(10),--总股数
   ltbl   number(10),--流通比例
   mgyl   number,--每股盈利
   jzc    number,--净资产
   jlr    number,--净利润
   zxsj   varchar2(2500),--最新事件
   bz     varchar2(2500)--备注说明
);

--板块目录
create table bkml(
   bkid  number primary key,
   bkmc  varchar2(100),
   bc    varchar2(100)
);

--每日实时数据
--注意：此表即使股票停牌，也有把最近的ＤＤＸ数据放入了
--写一个存储过程，生成每日的排名:参数是日期，注意那些停牌的哦，呵呵。。。。
create table daydata(
   gpdm  varchar2(6),
   rq     date,
   price  number,
   zf     number,
   ddx    number,
   ddy    number,
   ddz    number,
   ddx_sixty number,
   ddy_sixty number,
   ddx_tenl  number,
   ddx_tenc  number,
   tddc      number,
   ddc       number,
   zdc       number,
   xdc       number,
   hyd       number,
   dsb       number,
   tdmr      number,
   tdmc      number,
   ddmr      number,
   ddmc      number,
   zdmr      number,
   zdmc      number,
   xdmr      number,
   xdmc      number,
   hsl       number,
   lb        number,
   ddxpm     number,
   ddypm     number,
   ddzpm     number,
   constraint pk_daydata primary key(gpdm,rq)
);

--两市大单净买入全部排名
create table ddmrpm(
   gpdm  varchar2(6),
   rq    date,   
   pm    number,
   ddmr  number,
   ddmc  number,
   mrmcc number,
   cjl   number,
   jg    number,
   constraint pk_ddmrpm primary key(gpdm,rq)
);

--5日股票大单净买入排名50
create table fiveddmrpm(
   gpdm  varchar2(6),
   rq    date,   
   pm    number,
   ddmr  number,
   ddmc  number,
   mrmcc number,
   cjl   number,
   jg    number,
   constraint pk_fiveddmrpm primary key(gpdm,rq)
);

--10日股票大单净买入排名50
create table tenddmrpm(
   gpdm  varchar2(6),
   rq    date,   
   pm    number,
   ddmr  number,
   ddmc  number,
   mrmcc number,
   cjl   number,
   jg    number,
   constraint pk_tenddmrpm primary key(gpdm,rq)
);

--5日股票大单净卖出排名50
create table fiveddmcpm(
   gpdm  varchar2(6),
   rq    date,   
   pm    number,
   ddmr  number,
   ddmc  number,
   mrmcc number,
   cjl   number,
   jg    number,
   constraint pk_fiveddmcpm primary key(gpdm,rq)
);

--10日股票大单净卖出排名50
create table tenddmcpm(
   gpdm  varchar2(6),
   rq    date,   
   pm    number,
   ddmr  number,
   ddmc  number,
   mrmcc number,
   cjl   number,
   jg    number,
   constraint pk_tenddmcpm primary key(gpdm,rq)
);

--沪市Level-2数据 DDX 排名前50
--注意：此表如果股票停牌，没有把最近的ＤＤＸ数据放入了
create table ddxpm(
   gpdm  varchar2(6),
   rq    date,   
   jg    number,
   zf    number,
   ddx   number,
   ddxpm number,
   ten_phts  number,
   ten_lxphts   number,
   ddy    number,
   ddypm  number,
   ddz    number,
   ddzpm  number,
   constraint pk_ddxpm primary key(gpdm,rq)
);

--两市板块资金净流入排名
create table bkzjlrpm(
   bkid  number,
   pm    number,
   rq    date,  
   lr    number,
   lc    number,
   jlr   number,--尽流入
   gpdm  varchar2(20),--代表个股
   constraint pk_bkzjlrpm primary key(bkid,rq)
);

--5日两市板块资金净流入排名
create table fivebkzjlrpm(
   bkid  number,
   rq    date,
   pm    number, 
   lr    number,
   lc    number,
   jlr   number,--尽流入
   constraint pk_fivebkzjlrpm primary key(bkid,rq)
);

--10日两市板块资金净流入排名
create table tenbkzjlrpm(
   bkid  number,
   pm    number,
   rq    date,  
   lr    number,
   lc    number,
   jlr   number,--尽流入
   constraint pk_tenbkzjlrpm primary key(bkid,rq)
);

--主力持仓排名(最新更新：2009年5月26日)，把握主力资金的动向!
create table zlccpm(
   gpdm   varchar2(6),
   rq     date,
   adddec number,--增减性质 1代表增 0代表减
   dayNum number, --1日 5日 10日
   userType number,--主力分类：0主力 1特大户 2大户  3散户
   pm     number,--排名
   per    number,--增减百分比
   constraint pk_zlccpm primary key(gpdm,rq,adddec,dayNum,userType)   
)

 

 

       [用来进行排名的存储过程]

create or replace procedure produce_pm(in_Date IN daydata.rq%TYPE) is
  CURSOR cur_ddxpm is
    SELECT gpdm
      FROM daydata
      WHERE rq = trunc(in_Date)
       AND price > 0
     ORDER BY ddx desc,ddy desc,ddz desc;
    
  CURSOR cur_ddypm is
    SELECT gpdm
      FROM daydata
      WHERE rq = trunc(in_Date)
       AND price > 0
     ORDER BY ddy desc,ddx desc,ddz desc;    
    
  CURSOR cur_ddzpm is
    SELECT gpdm
      FROM daydata
      WHERE rq = trunc(in_Date)
       AND price > 0
     ORDER BY ddz desc,ddx desc,ddy desc;       
    
  curIndex NUMBER  :=0;
 
BEGIN
  FOR rec IN cur_ddxpm LOOP
    curIndex := curIndex + 1;
    update daydata set ddxpm=curIndex where gpdm=rec.gpdm and rq=trunc(in_date);
  END LOOP;
 
  curIndex:=0;
  FOR rec IN cur_ddypm LOOP
    curIndex := curIndex + 1;
    update daydata set ddypm=curIndex where gpdm=rec.gpdm and rq=trunc(in_date);
  END LOOP;
 
   
  curIndex:=0;
  FOR rec IN cur_ddzpm LOOP
    curIndex := curIndex + 1;
    update daydata set ddzpm=curIndex where gpdm=rec.gpdm and rq=trunc(in_date);
  END LOOP;
   
  commit;
end produce_pm;

 

 

    

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;

public class ConnectionManager {
    public static Connection getConnection() {
	Connection conn = null;
	try {
	    DriverManager.registerDriver(new oracle.jdbc.OracleDriver());

	    String url = "jdbc:oracle:oci8:@";
	    try {
		String url1 = System.getProperty("JDBC_URL");
		if (url1 != null)
		    url = url1;
	    } catch (Exception e) {
		e.printStackTrace();
	    }

	    conn = DriverManager.getConnection(url, "gp", "gp");
	} catch (Exception e1) {
	    e1.printStackTrace();
	}
	return conn;
    }
}

 

 

 

import java.awt.Dimension;
import java.awt.Toolkit;
import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlTable;
import com.gargoylesoftware.htmlunit.html.HtmlTableRow;


public class DataImport extends javax.swing.JFrame {
    String zlccDate = null;//主力持仓，赢富数据要慢一天

    /**
     * 构造函数
     */
    public DataImport() {
	initComponents();
	Calendar calendar = Calendar.getInstance();
	calendar.add(Calendar.DAY_OF_MONTH, -1);
	this.txtZlcc.setText(new SimpleDateFormat("yyyy-MM-dd").format(calendar.getTime()));
	this.txtDate.setText(new SimpleDateFormat("yyyy-MM-dd").format(new Date()));

	Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize();
	Dimension frameSize = this.getPreferredSize();

	if (frameSize.height > screenSize.height)
	    frameSize.height = screenSize.height;
	if (frameSize.width > screenSize.width)
	    frameSize.width = screenSize.width;

	this.setLocation((screenSize.width - frameSize.width) / 2, (screenSize.height - frameSize.height) / 2);
    }

    /** This method is called from within the constructor to
     * initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is
     * always regenerated by the Form Editor.
     */
    private void initComponents() {
	jPanel1 = new javax.swing.JPanel();
	btnImport = new javax.swing.JButton();
	txtDate = new javax.swing.JTextField();
	jScrollPane1 = new javax.swing.JScrollPane();
	txtStatus = new javax.swing.JTextArea();
	txtZlcc = new javax.swing.JTextField();
	jLabel1 = new javax.swing.JLabel();

	setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);

	btnImport.setText("\u5f00\u59cb\u5bfc\u5165\u6570\u636e");
	btnImport.addActionListener(new java.awt.event.ActionListener() {
	    public void actionPerformed(java.awt.event.ActionEvent evt) {
		btnImportActionPerformed(evt);
	    }
	});

	txtStatus.setColumns(20);
	txtStatus.setRows(5);
	jScrollPane1.setViewportView(txtStatus);

	jLabel1.setText("\u4e3b\u529b\u6301\u4ed3\u6392\u540d");

	org.jdesktop.layout.GroupLayout jPanel1Layout = new org.jdesktop.layout.GroupLayout(jPanel1);
	jPanel1.setLayout(jPanel1Layout);
	jPanel1Layout.setHorizontalGroup(jPanel1Layout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING).add(
		jPanel1Layout.createSequentialGroup().add(
			jPanel1Layout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING).add(jPanel1Layout.createSequentialGroup().addContainerGap().add(jScrollPane1, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, 425, Short.MAX_VALUE)).add(
				jPanel1Layout.createSequentialGroup().add(49, 49, 49).add(
					jPanel1Layout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING).add(btnImport).add(
						jPanel1Layout.createSequentialGroup().add(txtDate, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 136, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE).add(16, 16, 16).add(jLabel1).addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED).add(txtZlcc,
							org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 121, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))).add(37, 37, 37))).addContainerGap()));
	jPanel1Layout.setVerticalGroup(jPanel1Layout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING).add(
		jPanel1Layout.createSequentialGroup().addContainerGap().add(
			jPanel1Layout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE).add(txtDate, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE).add(jLabel1).add(txtZlcc,
				org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)).addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE).add(
			btnImport).addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED).add(jScrollPane1, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 352, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE).addContainerGap()));

	getContentPane().add(jPanel1, java.awt.BorderLayout.CENTER);

	pack();
    }// </editor-fold>

    private void btnImportActionPerformed(java.awt.event.ActionEvent evt) {
	SwingWorker<Object, Object> sw = new SwingWorker<Object, Object>() {
	    protected Object doInBackground() throws Exception {
		importData();
		return null;
	    }
	};
	sw.execute();
    }

    /**
     * 按钮事件，正式套入数据
     */
    private void importData() {
	String importDate = this.txtDate.getText();
	zlccDate=this.txtZlcc.getText();
	
	final WebClient webClient = new WebClient();

	//1 首先导入每日数据(来自散户大家庭)
	this.txtStatus.append("开始导入http://www.shdjt.com/中的每日数据........");
	try {
	    importDayData(webClient, importDate);
	    this.txtStatus.append("\n成功！");
	} catch (Exception e) {
	    e.printStackTrace();
	    this.txtStatus.append("\n失败！");
	    return;
	}
	this.txtStatus.append("\n");

	//2 当日板块、五日板块、十日板块资金流入
	this.txtStatus.append("\n开始导入当日板块、五日板块、十日板块资金流入........");
	try {
	    importBkData(webClient, zlccDate);
	    this.txtStatus.append("\n成功！");
	} catch (Exception e) {
	    e.printStackTrace();
	    this.txtStatus.append("\n失败！");
	    return;
	}
	this.txtStatus.append("\n");

	//3 两市大单净买入全部排名 当日、五日、十日
	this.txtStatus.append("\n开始导入两市大单净买入全部排名 当日、五日、十日........");
	try {
	    importDDmrpm(webClient, importDate);
	    this.txtStatus.append("\n成功！");
	} catch (Exception e) {
	    e.printStackTrace();
	    this.txtStatus.append("\n失败！");
	    return;
	}
	this.txtStatus.append("\n");

	//4 沪市Level-2数据 DDX 排名前50
	this.txtStatus.append("\n开始导入沪市Level-2数据 DDX 排名前50........");
	try {
	    importDDXpm(webClient, importDate);
	    this.txtStatus.append("\n成功！");
	} catch (Exception e) {
	    e.printStackTrace();
	    this.txtStatus.append("\n失败！");
	    return;
	}
	this.txtStatus.append("\n");

	//5 主力持仓排名
	this.txtStatus.append("\n开始导入主力持仓排名........");
	try {
	    importZlCCData(webClient, importDate);
	    this.txtStatus.append("\n成功！");
	} catch (Exception e) {
	    e.printStackTrace();
	    this.txtStatus.append("\n失败！");
	    return;
	}
	this.txtStatus.append("\n");

	//6 调用存储过程，对每日数据进行排名（DDX,DDY,DDZ）
	this.txtStatus.append("\n开始执行存储过程........");
	try {
	    callProcForPM(importDate);
	    this.txtStatus.append("\n成功！");
	} catch (Exception e) {
	    e.printStackTrace();
	    this.txtStatus.append("\n失败！");
	    return;
	}

	this.txtStatus.append("\n-----------------全部成功-----------------");

    }

    /**
     * 每日详细数据（散户大家庭）
     * @param webClient
     * @param importDate
     * @throws Exception
     */
    public void importDayData(WebClient webClient, String importDate) throws Exception {
	final HtmlPage page = webClient.getPage("http://www.shdjt.com/");
	final HtmlTable tableElement = (HtmlTable) page.getByXPath("//table[@bgcolor='#FFffff']").get(0);

	List<HtmlTableRow> list = tableElement.getRows();
	int size = list.size();

	List<HtmlTableRow> list2 = list.subList(2, size);

	PreparedStatement pstmt = null;
	Connection conn = null;
	Number a;
	NumberFormat nf = NumberFormat.getInstance();
	try {
	    conn = ConnectionManager.getConnection();
	    pstmt = conn.prepareStatement("insert into daydata(gpdm,rq,price,zf,ddx,ddy,ddz,ddx_sixty,ddy_sixty,ddx_tenl,ddx_tenc,tddc,ddc,zdc,xdc,hyd,dsb,tdmr,tdmc,ddmr,ddmc,zdmr,zdmc,xdmr,xdmc,hsl,lb) " + "values (?,to_date('" + importDate
		    + "','yyyy-MM-dd'),?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");

	    for (final HtmlTableRow row : list2) {
		pstmt.setString(1, isNull(row.getCell(3).asText()));// 股票代码
		pstmt.setFloat(2, Float.valueOf(isNull(row.getCell(5).asText())).floatValue());// 价格
		pstmt.setFloat(3, Float.valueOf(isNull(row.getCell(6).asText())).floatValue());// 涨幅
		pstmt.setFloat(4, Float.valueOf(isNull(row.getCell(7).asText())).floatValue());//
		pstmt.setFloat(5, Float.valueOf(isNull(row.getCell(8).asText())).floatValue());//
		pstmt.setFloat(6, nf.parse(row.getCell(9).asText()).floatValue());//DDZ
		pstmt.setFloat(7, Float.valueOf(isNull(row.getCell(10).asText())).floatValue());//
		pstmt.setFloat(8, Float.valueOf(isNull(row.getCell(11).asText())).floatValue());//
		pstmt.setInt(9, Integer.valueOf(isNull(row.getCell(12).asText())).intValue());// 10日次数
		pstmt.setInt(10, Integer.valueOf(isNull(row.getCell(13).asText())).intValue());// 10日次数连续
		pstmt.setFloat(11, Float.valueOf(isNull(row.getCell(14).asText())).floatValue());//
		pstmt.setFloat(12, Float.valueOf(isNull(row.getCell(15).asText())).floatValue());//
		pstmt.setFloat(13, Float.valueOf(isNull(row.getCell(16).asText())).floatValue());//
		pstmt.setFloat(14, Float.valueOf(isNull(row.getCell(17).asText())).floatValue());//
		pstmt.setInt(15, Integer.valueOf(isNull(row.getCell(18).asText())).intValue());// 活跃度
		pstmt.setFloat(16, Float.valueOf(isNull(row.getCell(19).asText())).floatValue());//
		pstmt.setFloat(17, Float.valueOf(isNull(row.getCell(20).asText())).floatValue());//
		pstmt.setFloat(18, Float.valueOf(isNull(row.getCell(21).asText())).floatValue());//
		pstmt.setFloat(19, Float.valueOf(isNull(row.getCell(22).asText())).floatValue());//
		pstmt.setFloat(20, Float.valueOf(isNull(row.getCell(23).asText())).floatValue());//
		pstmt.setFloat(21, Float.valueOf(isNull(row.getCell(24).asText())).floatValue());//
		pstmt.setFloat(22, Float.valueOf(isNull(row.getCell(25).asText())).floatValue());//
		pstmt.setFloat(23, Float.valueOf(isNull(row.getCell(26).asText())).floatValue());//
		pstmt.setFloat(24, Float.valueOf(isNull(row.getCell(27).asText())).floatValue());//
		pstmt.setFloat(25, Float.valueOf(isNull(row.getCell(28).asText())).floatValue());//
		pstmt.setFloat(26, Float.valueOf(isNull(row.getCell(29).asText())).floatValue());//

		pstmt.addBatch();
	    }
	    pstmt.executeBatch();
	    conn.commit();
	} finally {
	    try {
		pstmt.close();
		conn.close();
	    } catch (SQLException e) {
		e.printStackTrace();
	    }
	}
    }

    public static String isNull(String s) {
	String p = "0.0";
	int leng = s.length();

	if (s != null && leng > 0)
	    p = s;

	return p;
    }

    /**
     * 当日板块、五日板块、十日板块资金流入
     * @param webClient
     * @param currentDate
     */
    public void importBkData(WebClient webClient, String currentDate) throws Exception {
	importBkzjData(webClient, "http://www.gupiaodadan.cn/bankuai.jsp", "bkzjlrpm", currentDate);
	importBkzjData(webClient, "http://www.gupiaodadan.cn/bankuaimore-2", "fivebkzjlrpm", currentDate);
	importBkzjData(webClient, "http://www.gupiaodadan.cn/bankuaimore-1", "tenbkzjlrpm", currentDate);
    }

    public void importBkzjData(WebClient webClient, String url, String tableName, String date) throws Exception {
	PreparedStatement pstmt = null;
	Connection conn = null;

	try {
	    final HtmlPage page = webClient.getPage(url);
	    final HtmlDivision div = (HtmlDivision) page.getByXPath("//div[@class='tablecontent']").get(0);

	    HtmlTable tableElement = (HtmlTable) div.getHtmlElementsByTagName("table").get(0);

	    List<HtmlTableRow> list = tableElement.getRows();
	    int size = list.size();

	    List<HtmlTableRow> list2 = list.subList(1, size);

	    conn = ConnectionManager.getConnection();
	    if (tableName.equalsIgnoreCase("bkzjlrpm"))
		pstmt = conn.prepareStatement("insert into " + tableName + "(bkid,rq,pm,lr,lc,jlr,gpdm) values (?,to_date('" + date + "','yyyy-MM-dd'),?,?,?,?,?)");
	    else
		pstmt = conn.prepareStatement("insert into " + tableName + "(bkid,rq,pm,lr,lc,jlr) values (?,to_date('" + date + "','yyyy-MM-dd'),?,?,?,?)");

	    for (final HtmlTableRow row : list2) {
		int pm = Integer.valueOf(row.getCell(0).asText());// 排名

		HtmlElement tdElement = row.getHtmlElementsByTagName("td").get(1);
		HtmlElement aElement = tdElement.getHtmlElementsByTagName("a").get(0);
		String strUrl = aElement.getAttribute("href");
		int startIndex = strUrl.lastIndexOf("-");
		String bkId = strUrl.substring(startIndex + 1, strUrl.length());

		float lr = Float.valueOf(row.getCell(2).asText()).floatValue();
		float lc = Float.valueOf(row.getCell(3).asText()).floatValue();
		float jlr = Float.valueOf(row.getCell(4).asText()).floatValue();

		pstmt.setInt(1, Integer.valueOf(bkId).intValue());
		pstmt.setInt(2, pm);
		pstmt.setFloat(3, lr);
		pstmt.setFloat(4, lc);
		pstmt.setFloat(5, jlr);

		if (tableName.equalsIgnoreCase("bkzjlrpm")) {
		    tdElement = row.getHtmlElementsByTagName("td").get(5);
		    aElement = tdElement.getHtmlElementsByTagName("a").get(0);
		    strUrl = aElement.getAttribute("href");
		    String gpdm = strUrl.substring(strUrl.length() - 6);
		    pstmt.setString(6, gpdm);
		}

		pstmt.addBatch();
	    }
	    pstmt.executeBatch();
	    conn.commit();
	} finally {
	    try {
		pstmt.close();
		conn.close();
	    } catch (SQLException e) {
		e.printStackTrace();
	    }
	}
    }

    /**
     * 两市大单净买入全部排名 当日、五日、十日
     * @param webClient
     * @param currentDate
     * @throws Exception
     */
    public void importDDmrpm(WebClient webClient, String currentDate) throws Exception {
	// 当日买卖排名
	String currentURL = "http://www.gupiaodadan.cn/all.jsp";
	importDdMrData(webClient, currentURL, "ddmrpm", currentDate);

	// 五日买入
	String fivemrURL = "http://www.gupiaodadan.cn/more-3";
	importDdMrData(webClient, fivemrURL, "fiveddmrpm", currentDate);

	// 十日买入
	String tenmrURL = "http://www.gupiaodadan.cn/more-4";
	importDdMrData(webClient, tenmrURL, "tenddmrpm", currentDate);

	// 五日卖出
	String fivemcURL = "http://www.gupiaodadan.cn/more-5";
	importDdMrData(webClient, fivemcURL, "fiveddmcpm", currentDate);

	// 十日卖出
	String tenmcURL = "http://www.gupiaodadan.cn/more-6";
	importDdMrData(webClient, tenmcURL, "tenddmcpm", currentDate);
    }

    public static void importDdMrData(WebClient webClient, String url, String tableName, String date) throws Exception {
	PreparedStatement pstmt = null;
	Connection conn = null;
	NumberFormat nf = NumberFormat.getInstance();

	try {
	    final HtmlPage page = webClient.getPage(url);
	    final HtmlDivision div = (HtmlDivision) page.getByXPath("//div[@class='tablecontent']").get(0);

	    HtmlTable tableElement = (HtmlTable) div.getHtmlElementsByTagName("table").get(0);

	    List<HtmlTableRow> list = tableElement.getRows();
	    int size = list.size();

	    List<HtmlTableRow> list2 = list.subList(1, size);

	    conn = ConnectionManager.getConnection();
	    pstmt = conn.prepareStatement("insert into " + tableName + "(gpdm,rq,pm,ddmr,ddmc,mrmcc,cjl,jg) values (?,to_date('" + date + "','yyyy-MM-dd'),?,?,?,?,?,?)");
	    for (final HtmlTableRow row : list2) {
		int pm = Integer.valueOf(row.getCell(0).asText());
		String gpdm = row.getCell(1).asText();

		int ddmr = nf.parse(row.getCell(3).asText()).intValue();
		int ddmc = nf.parse(row.getCell(4).asText()).intValue();
		int mrmcc = ddmr - ddmc;
		int cjl = nf.parse(row.getCell(5).asText()).intValue();
		float jg = Float.valueOf(row.getCell(6).asText()).floatValue();

		pstmt.setString(1, gpdm);
		pstmt.setInt(2, pm);
		pstmt.setInt(3, ddmr);
		pstmt.setInt(4, ddmc);
		pstmt.setInt(5, mrmcc);
		pstmt.setInt(6, cjl);
		pstmt.setFloat(7, jg);

		pstmt.addBatch();
	    }
	    pstmt.executeBatch();
	    conn.commit();
	} finally {
	    try {
		pstmt.close();
		conn.close();
	    } catch (SQLException e) {
		e.printStackTrace();
	    }
	}
    }

    /**
     * 沪市Level-2数据 DDX 排名前50
     * @param webClient
     * @param curDate
     * @throws Exception
     */
    public void importDDXpm(WebClient webClient, String curDate) throws Exception {

	final HtmlPage page = webClient.getPage("http://www.gupiaodadan.cn/level2-home-1");
	final HtmlDivision div = (HtmlDivision) page.getByXPath("//div[@class='tablecontent']").get(0);

	HtmlTable tableElement = (HtmlTable) div.getHtmlElementsByTagName("table").get(0);

	List<HtmlTableRow> list = tableElement.getRows();
	int size = list.size();

	List<HtmlTableRow> list2 = list.subList(2, size);

	PreparedStatement pstmt = null;
	Connection conn = null;

	try {
	    conn = ConnectionManager.getConnection();
	    pstmt = conn.prepareStatement("insert into ddxpm( gpdm,rq,jg,zf,ddx,ddxpm,ten_phts,ten_lxphts,ddy,ddypm,ddz,ddzpm) " + "values (?,to_date('" + curDate + "','yyyy-MM-dd'),?,?,?,?,?,?,?,?,?,?)");
	    for (final HtmlTableRow row : list2) {
		pstmt.setString(1, row.getCell(0).asText());
		pstmt.setFloat(2, Float.valueOf(row.getCell(2).asText()).floatValue());
		pstmt.setFloat(3, Float.valueOf(row.getCell(3).asText()).floatValue());
		pstmt.setFloat(4, Float.valueOf(row.getCell(4).asText()).floatValue());
		pstmt.setInt(5, Integer.valueOf(row.getCell(5).asText()).intValue());
		pstmt.setInt(6, Integer.valueOf(row.getCell(6).asText()).intValue());
		pstmt.setInt(7, Integer.valueOf(row.getCell(7).asText()).intValue());
		pstmt.setFloat(8, Float.valueOf(row.getCell(8).asText()).floatValue());
		pstmt.setInt(9, Integer.valueOf(row.getCell(9).asText()).intValue());
		pstmt.setFloat(10, Float.valueOf(row.getCell(10).asText()).floatValue());
		pstmt.setInt(11, Integer.valueOf(row.getCell(11).asText()).intValue());
		pstmt.addBatch();
	    }
	    pstmt.executeBatch();
	    conn.commit();
	} finally {
	    try {
		pstmt.close();
		conn.close();
	    } catch (SQLException e) {

	    }
	}
    }

    /**
     * 主力持仓排名
     * @param webClient
     * @param sDate
     * @throws Exception
     */
    public static void importZlCCData(WebClient webClient, String sDate) throws Exception {
	PreparedStatement pstmt = null;
	Connection conn = null;

	try {
	    conn = ConnectionManager.getConnection();
	    pstmt = conn.prepareStatement("insert into zlccpm(gpdm,rq,adddec,dayNum,userType,pm,per) " + "values (?,to_date('" + sDate + "','yyyy-MM-dd'),?,?,?,?,?)");

	    final HtmlPage page = webClient.getPage("http://www.gupiaodadan.cn/topview-search.jsp");

	    List<?> listDivs = page.getByXPath("//div[@class=\"fr02\"]");

	    //adddec number,--增减性质 1代表增 0代表减
	    //dayNum number, --1代表1日 5代表5日 10代表10日
	    //userType number,--主力分类：0主力 1特大户 2大户  3散户

	    // 1日主力增仓
	    HtmlDivision div = (HtmlDivision) listDivs.get(0);
	    processData(div, pstmt, 1, 1, 0);

	    // 1日特大户增仓
	    div = (HtmlDivision) listDivs.get(1);
	    processData(div, pstmt, 1, 1, 1);

	    // 1日大户增仓
	    div = (HtmlDivision) listDivs.get(2);
	    processData(div, pstmt, 1, 1, 2);

	    // 1日散户减仓
	    div = (HtmlDivision) listDivs.get(3);
	    processData(div, pstmt, 0, 1, 3);

	    //-------------------------------------------
	    // 5日主力增仓
	    div = (HtmlDivision) listDivs.get(4);
	    processData(div, pstmt, 1, 5, 0);

	    // 5日特大户增仓
	    div = (HtmlDivision) listDivs.get(5);
	    processData(div, pstmt, 1, 5, 1);

	    // 5日大户增仓
	    div = (HtmlDivision) listDivs.get(6);
	    processData(div, pstmt, 1, 5, 2);

	    // 5日散户减仓
	    div = (HtmlDivision) listDivs.get(7);
	    processData(div, pstmt, 0, 5, 3);

	    //-------------------------------------------
	    // 10日主力增仓
	    div = (HtmlDivision) listDivs.get(8);
	    processData(div, pstmt, 1, 10, 0);

	    // 10日特大户增仓
	    div = (HtmlDivision) listDivs.get(9);
	    processData(div, pstmt, 1, 10, 1);

	    // 10日大户增仓
	    div = (HtmlDivision) listDivs.get(10);
	    processData(div, pstmt, 1, 10, 2);

	    // 10日散户减仓
	    div = (HtmlDivision) listDivs.get(11);
	    processData(div, pstmt, 0, 10, 3);

	    pstmt.executeBatch();
	    conn.commit();
	} finally {
	    try {
		pstmt.close();
		conn.close();
	    } catch (SQLException e) {
		e.printStackTrace();
	    }
	}
    }

    public static void processData(HtmlDivision div, PreparedStatement pstmt, int adddec, int dayNum, int userType) throws Exception {
	HtmlTable tableElement = (HtmlTable) div.getHtmlElementsByTagName("table").get(0);

	List<HtmlTableRow> list = tableElement.getRows();
	int size = list.size();

	List<HtmlTableRow> list2 = list.subList(1, size);

	// gpdm,rq,adddec,dayNum,userType,pm,per
	for (final HtmlTableRow row : list2) {
	    pstmt.setInt(5, Integer.parseInt(row.getCell(0).asText()));// 排名

	    HtmlElement tdElement = row.getHtmlElementsByTagName("td").get(1);
	    HtmlElement aElement = tdElement.getHtmlElementsByTagName("a").get(0);
	    String strUrl = aElement.getAttribute("href");
	    int startIndex = strUrl.lastIndexOf("-");
	    String gpdm = strUrl.substring(startIndex + 1, strUrl.length());

	    pstmt.setString(1, gpdm);
	    pstmt.setInt(2, adddec);
	    pstmt.setInt(3, dayNum);
	    pstmt.setInt(4, userType);
	    pstmt.setFloat(6, Float.parseFloat(row.getCell(2).asText()));//百分比

	    pstmt.addBatch();
	}
    }

    private void callProcForPM(String strDate) throws Exception {
	Connection conn = ConnectionManager.getConnection();
	CallableStatement cs = null;
	try {
	    cs = conn.prepareCall("{call produce_pm(?)}");
	    cs.setDate(1, new java.sql.Date(new SimpleDateFormat("yyyy-MM-dd").parse(strDate).getTime()));
	    cs.executeUpdate();
	} finally {
	    try {
		cs.close();
		conn.close();
	    } catch (SQLException e) {
		e.printStackTrace();
	    }
	}
    }

    /**
     * @param args the command line arguments
     */
    public static void main(String args[]) {
	java.awt.EventQueue.invokeLater(new Runnable() {
	    public void run() {
		new DataImport().setVisible(true);
	    }
	});
    }

    //GEN-BEGIN:variables
    // Variables declaration - do not modify
    private javax.swing.JButton btnImport;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JPanel jPanel1;
    private javax.swing.JScrollPane jScrollPane1;
    private javax.swing.JTextField txtDate;
    private javax.swing.JTextArea txtStatus;
    private javax.swing.JTextField txtZlcc;
    // End of variables declaration//GEN-END:variables
}

 

   

    

评论

1 楼 Heppy 2013-08-29  

向楼主的无私奉献致敬