Thursday, August 30, 2012

ZK Pivottable: Handling huge data in ZK Pivottable



Introduction

In the previous post http://ben-bai.blogspot.tw/2012/07/zk-pivottable-display-data-in-zk.html, we load all data into pivot model at once, in this case the memory consumption may be a problem if the raw data is huge.

In this post, we will try to handle data paging and field control by ourself to load data partially, for the sake of reducing memory consumption. However this may need some additional processing time which would be a trade-off but is useful if the data is really huge.

The Program

PartialRenderComposer.java

package test;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;

import org.zkoss.pivot.PivotField;
import org.zkoss.pivot.Pivottable;
import org.zkoss.pivot.impl.TabularPivotModel;

import org.zkoss.zk.ui.Component;
import org.zkoss.zk.ui.event.InputEvent;
import org.zkoss.zk.ui.event.SelectEvent;
import org.zkoss.zk.ui.util.GenericForwardComposer;
import org.zkoss.zkmax.zul.Chosenbox;
import org.zkoss.zul.Intbox;
import org.zkoss.zul.ListModelList;
import org.zkoss.zul.Timer;

/**
 * The scenario is to reduce memory consumption, get only required
 * data from data base each time and purge it after the data for pivot
 * model are ready
 * 
 * @author ben
 *
 */
public class PartialRenderComposer extends GenericForwardComposer {
    private Pivottable pivottable; // pitottable
    private Intbox paging; // custom paging control
    private Intbox info;
    private Intbox nodes;
    private Intbox memInfo;
    private Chosenbox rowFieldChosen;
    private Chosenbox columnFieldChosen;
    private Chosenbox dataFieldChosen;
    private Timer checkTimer;

    private TabularPivotModel _pivotModel;

    private Runtime rt = Runtime.getRuntime();

    // fake test data, render only once
    private static List<DataClass> _testData; // data that simulate a db resource
    private List<String> _rowFieldList;
    private List<String> _columnFieldList;
    private List<String> _dataFieldList;
    private List<String> _unusedFieldList;
    private int _size = 0;
    private int _currentPage = 1; // page of custom paging
    private int _pageNodesLimit = 10; // number of first level node of a page
    private int _testDataSize = 200000;

    // ------------------------------- //
    // flow control                    //
    // ------------------------------- //
    @SuppressWarnings("unchecked")
    public void doAfterCompose (Component comp) throws Exception {
        super.doAfterCompose(comp);
        info.setValue(_size);

        updateFieldLists();

        nodes.setValue(_pageNodesLimit);
        // show all data in one page
        pivottable.setPageSize(_size);
    }

    // ------------------------------- //
    // Event handling                  //
    // ------------------------------- //
    /**
     * update page number then update pivot model
     */
    public void onChanging$paging (InputEvent e) {
        _currentPage = Integer.parseInt(e.getValue());
        if ((_currentPage-1) * _pageNodesLimit >= 10) {
            // return to first page if overflow
            _currentPage = 1;
            paging.setValue(_currentPage);
        }
        updatePivotModel();
    }
    /**
     * update first-level-nodes per page then update pivot model
     * @param e
     */
    public void onChange$nodes (InputEvent e) {
        _pageNodesLimit = Integer.parseInt(e.getValue());
        updatePivotModel();
    }

    /**
     * update field list and data then update models
     * @param e
     */
    public void onSelect$rowFieldChosen (SelectEvent e) {
        moveItem(e, _rowFieldList, _unusedFieldList);
        updatePivotModel();
        updateFieldLists();
    }
    /**
     * update field list and data then update models
     * @param e
     */
    public void onSelect$columnFieldChosen (SelectEvent e) {
        moveItem(e, _columnFieldList, _unusedFieldList);
        updatePivotModel();
        updateFieldLists();
    }
    /**
     * update field list and data then update models
     * @param e
     */
    public void onSelect$dataFieldChosen (SelectEvent e) {
        moveItem(e, _dataFieldList, _unusedFieldList);
        updatePivotModel();
        updateFieldLists();
    }
    /**
     * require gc and update memory info
     */
    public void onTimer$checkTimer () {
        System.gc();
        memInfo.setValue(new Long((rt.totalMemory()-rt.freeMemory())/1024/1024).intValue());
    }

    // ------------------------------- //
    // data management                 //
    // ------------------------------- //
    public TabularPivotModel getPivotModel () throws Exception {
        List<List<Object>> rawData = getPivotData();
        _pivotModel = new TabularPivotModel(rawData, getColumns());
        List<String> l;
        int i;

        for (i = 0, l = getRowFieldList(); i < l.size(); i++) {
            _pivotModel.setFieldType(l.get(i), PivotField.Type.ROW);
        }
        for (i = 0, l = getColumnFieldList(); i < l.size(); i++) {
            _pivotModel.setFieldType(l.get(i), PivotField.Type.COLUMN);
        }
        for (i = 0, l = getDataFieldList(); i < l.size(); i++) {
            _pivotModel.setFieldType(l.get(i), PivotField.Type.DATA);
        }
        return _pivotModel;
    }
    /**
     * update pivot model
     */
    private void updatePivotModel() {
        try {
            _pivotModel = getPivotModel();
            pivottable.setModel(_pivotModel);
        } catch (Exception e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        }
    }
    /**
     * update chosenbox field list
     */
    private void updateFieldLists () {
        ListModelList model = getChosenModel(getRowFieldList(), getUnusedFieldList());
        rowFieldChosen.setModel(model);
        model.setSelection(_rowFieldList);

        model = getChosenModel(getColumnFieldList(), getUnusedFieldList());
        columnFieldChosen.setModel(model);
        model.setSelection(_columnFieldList);

        model = getChosenModel(getDataFieldList(), getUnusedFieldList());
        dataFieldChosen.setModel(model);
        model.setSelection(_dataFieldList);
    }

    /**
     * Get partial data for pivot model
     * @return
     * @throws Exception
     */
    private List<List<Object>> getPivotData() throws Exception {
        List<List<Object>> l = new ArrayList<List<Object>>();
        l.addAll(getData());

        // update size info
        _size = l.size();
        if (info != null) {
            info.setValue(_size);
        }
        if (pivottable != null) {
            // show all data in one page
            pivottable.setPageSize(_size);
        }

        _testData.clear();
        _testData = null;
        return l;
    }
    /**
     * Move field from specific field list to unused field list ((removed))
     * or vice versa (added)
     * @param e
     * @param origin
     * @param unuse
     */
    private void moveItem (SelectEvent e, List origin, List unuse) {
        Object target = null;
        List objs = new ArrayList();
        boolean add = false;
        
        objs.addAll(e.getSelectedObjects());
        // check whether field removed
        for (Object obj : origin) {
            if (!objs.contains(obj)) {
                target = obj;
                break;
            }
        }
        // check whether field added
        if (target == null) {
            add = true;
            for (Object obj : unuse) {
                if (objs.contains(obj)) {
                    target = obj;
                    break;
                }
            }
        }
        if (add) {
            // field added, move it to self list
            origin.add((String)target);
            unuse.remove(target);
        } else {
            // field removed, move it to unused
            unuse.add((String)target);
            origin.remove(target);
        }
    }
    /**
     * Get the data list from fake db and take only required data,
     * for pivot model, based on the 'used' fields and data range
     * 
     * @return
     * @throws Exception
     */
    private List<List<Object>> getData() throws Exception {
        List<List<Object>> rawData = new ArrayList<List<Object>>();
        _rowFieldList = getRowFieldList(); // current row fields
        _columnFieldList = getColumnFieldList(); // current column fields
        _dataFieldList = getDataFieldList(); // current data fields
        // size for inner object array
        int arraySize = _rowFieldList.size() + _columnFieldList.size() + _dataFieldList.size();
        int index = 0;
        // data range
        int first = (_currentPage - 1)*_pageNodesLimit + 1;
        int last = (_currentPage - 1)*_pageNodesLimit + _pageNodesLimit;

        prepareTestData(); // prepare the test data
        for (DataClass d : _testData) {
            Object[] objs = new Object[arraySize];

            // check whether is in range
            String rowHead = (String)getValue(d, _rowFieldList.get(0));
            int nodeNo = Integer.parseInt((rowHead).substring(rowHead.indexOf("__") + 2, rowHead.length()));
            // Only load rows are in page range
            if (nodeNo >= first && nodeNo <= last) {
                // add values to array
                for (int i = 0; i < _rowFieldList.size(); i++) {
                    objs[index] = getValue(d, _rowFieldList.get(i));
                    index++;
                }
                for (int i = 0; i < _columnFieldList.size(); i++) {
                    objs[index] = getValue(d, _columnFieldList.get(i));
                    index++;
                }
                for (int i = 0; i < _dataFieldList.size(); i++) {
                    objs[index] = getValue(d, _dataFieldList.get(i));
                    index++;
                }
                index = 0; // reset index
                // add array to rawData
                rawData.add(Arrays.asList(objs));
            }
        }

        return rawData;
    }
    /**
     * Get value from DataClass with respect to the field name
     * @param d DataClass object
     * @param name field name
     * @return
     */
    private Object getValue (DataClass d, String name) {
        return "Row_One".equals(name)? d.getRowOne()
                : "Row_Two".equals(name)? d.getRowTwo()
                : "Row_Three".equals(name)? d.getRowThree()
                : "Column_One".equals(name)? d.getColumnOne()
                : "Column_Two".equals(name)? d.getColumnTwo()
                : "Column_Three".equals(name)? d.getColumnThree()
                : "Data_One".equals(name)? d.getDataOne()
                : "Data_Two".equals(name)? d.getDataTwo() : d.getDataThree();
    }
    /**
     * Fake db's data,
     * will be cleared after the data for PivotModel are ready
     */
    private void prepareTestData () {
        if (_testData == null) { // generate random data
            _testData = new ArrayList<DataClass>();
            Random r = new Random();
            for (int i = 0; i < _testDataSize; i++) {
                _testData.add(new DataClass(
                        "Row_One__" + (r.nextInt(10) + 1), "Row_Two__" + (r.nextInt(10) + 1), "Row_Three__" + (r.nextInt(10) + 1),
                        "Column_One__" + (r.nextInt(3) + 1), "Column_Two__" + (r.nextInt(3) + 1), "Column_Three__" + (r.nextInt(3) + 1),
                        r.nextInt(1000) + 1, r.nextInt(1000) + 1, r.nextInt(1000) + 1));
            }
        }
    }
    /**
     * Get the field names for pivottable
     * @return
     */
    private List<String> getColumns() {
        List<String> l = new ArrayList<String>();
        l.addAll(getRowFieldList());
        l.addAll(getColumnFieldList());
        l.addAll(getDataFieldList());
        return l;
    }
    /**
     * Get all row field names
     * @return
     */
    private List<String> getRowFieldList () {
        if (_rowFieldList == null) { // init
            _rowFieldList = new ArrayList<String>();
            _rowFieldList.add("Row_One");
            _rowFieldList.add("Row_Two");
            _rowFieldList.add("Row_Three");
        }
        return _rowFieldList;
    }
    /**
     * Get all column field names
     * @return
     */
    private List<String> getColumnFieldList () {
        if (_columnFieldList == null) { // init
            _columnFieldList = new ArrayList<String>();
            _columnFieldList.add("Column_One");
            _columnFieldList.add("Column_Two");
            _columnFieldList.add("Column_Three");
        }
        return _columnFieldList;
    }
    /**
     * Get all data field names
     * @return
     */
    private List<String> getDataFieldList () {
        if (_dataFieldList == null) { // init
            _dataFieldList = new ArrayList<String>();
            _dataFieldList.add("Data_One");
            _dataFieldList.add("Data_Two");
            _dataFieldList.add("Data_Three");
        }
        return _dataFieldList;
    }
    /**
     * Get all unused field names,
     * for chosenbox model only
     * @return
     */
    private List<String> getUnusedFieldList () {
        if (_unusedFieldList == null) { // init
            _unusedFieldList = new ArrayList<String>();
        }
        return _unusedFieldList;
    }
    /**
     * Get the list for chosenbox model by specific list (row/column/data)
     * plus unusedFieldList
     * @param fields The specific list and unusedFieldList
     * @return
     */
    private ListModelList<String> getChosenModel (List<String>... fields) {
        List<String> l = new ArrayList<String>(); 
        for (List<String> field : fields) {
            l.addAll(field);
        }
        return new ListModelList<String>(l);
    }
}


In this composer, we update the pivot model with respect to the current page, (first level) node amount per page and required fields. Only the data that will be shown are loaded into pivot model.

There is a timer to perform gc and update the value of memory consumption.

DataClass.java

package test;

public class DataClass {
    String _rowOne;
    String _rowTwo;
    String _rowThree;

    String _columnOne;
    String _columnTwo;
    String _columnThree;

    int _dataOne;
    int _dataTwo;
    int _dataThree;

    public DataClass (String rowOne, String rowTwo, String rowThree,
                    String columnOne, String columnTwo, String columnThree,
                    int dataOne, int dataTwo, int dataThree) {
        _rowOne = rowOne;
        _rowTwo = rowTwo;
        _rowThree = rowThree;
        _columnOne = columnOne;
        _columnTwo = columnTwo;
        _columnThree = columnThree;
        _dataOne = dataOne;
        _dataTwo = dataTwo;
        _dataThree = dataThree;
    }
    public String getRowOne() {
        return _rowOne;
    }
    public void setRowOne(String _rowOne) {
        this._rowOne = _rowOne;
    }
    public String getRowTwo() {
        return _rowTwo;
    }
    public void setRowTwo(String _rowTwo) {
        this._rowTwo = _rowTwo;
    }
    public String getRowThree() {
        return _rowThree;
    }
    public void setRowThree(String _rowThree) {
        this._rowThree = _rowThree;
    }
    public String getColumnOne() {
        return _columnOne;
    }
    public void setColumnOne(String _columnOne) {
        this._columnOne = _columnOne;
    }
    public String getColumnTwo() {
        return _columnTwo;
    }
    public void setColumnTwo(String _columnTwo) {
        this._columnTwo = _columnTwo;
    }
    public String getColumnThree() {
        return _columnThree;
    }
    public void setColumnThree(String _columnThree) {
        this._columnThree = _columnThree;
    }
    public int getDataOne() {
        return _dataOne;
    }
    public void setDataOne(int _dataOne) {
        this._dataOne = _dataOne;
    }
    public int getDataTwo() {
        return _dataTwo;
    }
    public void setDataTwo(int _dataTwo) {
        this._dataTwo = _dataTwo;
    }
    public int getDataThree() {
        return _dataThree;
    }
    public void setDataThree(int _dataThree) {
        this._dataThree = _dataThree;
    }
}


index.zul

<zk>
    <window id="win" xmlns:w="client"
        apply="test.PartialRenderComposer">
        <!-- custom paging that control data loading partially -->
        Page: <intbox value="1" id="paging" />
        <!-- how many data under current settings -->
        Pivot model data size: <intbox id="info" />
        <div></div>

        Nodes limit: <intbox id="nodes" />
        Memory used (MB): <intbox id="memInfo" />
        <div></div>

        Rows: <chosenbox id="rowFieldChosen" width="300px" />
        Columns: <chosenbox id="columnFieldChosen" width="300px" />
        <div></div>
        Datas: <chosenbox id="dataFieldChosen" width="300px" />
        <vlayout>
            <!-- the pageSize denotes how many rows to display in one page,
                you can set it as data size if you want display all data in one page -->
            <pivottable id="pivottable" model="${win$composer.pivotModel}"
                pageSize="30"
                onPivotNodeOpen="" />
        </vlayout>
        <timer id="checkTimer" delay="1000" repeats="true" />
    </window>
</zk>


The Result

View demo on line
http://screencast.com/t/aP0VwNVxUZ

Download

The full project
https://github.com/benbai123/ZK_Practice/tree/master/Components/projects/Addon_Practice/PivottableTest/ShowPartialDataInPivottable

Demo flash
https://github.com/benbai123/ZK_Practice/blob/master/Components/demos/addon/ShowPartialDataInPivottable.swf

References

http://books.zkoss.org/wiki/ZK_Pivottable_Essentials

No comments:

Post a Comment