daal utils printNumericTable

#===============================================================================
# Copyright 2014-2017 Intel Corporation
# All Rights Reserved.
#
# If this  software was obtained  under the  Intel Simplified  Software License,
# the following terms apply:
#
# The source code,  information  and material  ("Material") contained  herein is
# owned by Intel Corporation or its  suppliers or licensors,  and  title to such
# Material remains with Intel  Corporation or its  suppliers or  licensors.  The
# Material  contains  proprietary  information  of  Intel or  its suppliers  and
# licensors.  The Material is protected by  worldwide copyright  laws and treaty
# provisions.  No part  of  the  Material   may  be  used,  copied,  reproduced,
# modified, published,  uploaded, posted, transmitted,  distributed or disclosed
# in any way without Intel's prior express written permission.  No license under
# any patent,  copyright or other  intellectual property rights  in the Material
# is granted to  or  conferred  upon  you,  either   expressly,  by implication,
# inducement,  estoppel  or  otherwise.  Any  license   under such  intellectual
# property rights must be express and approved by Intel in writing.
#
# Unless otherwise agreed by Intel in writing,  you may not remove or alter this
# notice or  any  other  notice   embedded  in  Materials  by  Intel  or Intel's
# suppliers or licensors in any way.
#
#
# If this  software  was obtained  under the  Apache License,  Version  2.0 (the
# "License"), the following terms apply:
#
# You may  not use this  file except  in compliance  with  the License.  You may
# obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless  required  by   applicable  law  or  agreed  to  in  writing,  software
# distributed under the License  is distributed  on an  "AS IS"  BASIS,  WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the   License  for the   specific  language   governing   permissions  and
# limitations under the License.
#===============================================================================

## <a name="DAAL-EXAMPLE-PY-UTIL"></a>

from __future__ import print_function

import sys
import numpy as np

from daal.data_management import (
    CSRNumericTable, NumericTableIface, readOnly, BlockDescriptor, packed_mask,
    DataSourceIface, FileDataSource, HomogenTensor, SubtensorDescriptor
)

CRC_REM = [
    0x00000000, 0x741B8CD6, 0xE83719AC, 0x9C2C957A, 0xA475BF8E, 0xD06E3358, 0x4C42A622, 0x38592AF4,
    0x3CF0F3CA, 0x48EB7F1C, 0xD4C7EA66, 0xA0DC66B0, 0x98854C44, 0xEC9EC092, 0x70B255E8, 0x04A9D93E,
    0x79E1E794, 0x0DFA6B42, 0x91D6FE38, 0xE5CD72EE, 0xDD94581A, 0xA98FD4CC, 0x35A341B6, 0x41B8CD60,
    0x4511145E, 0x310A9888, 0xAD260DF2, 0xD93D8124, 0xE164ABD0, 0x957F2706, 0x0953B27C, 0x7D483EAA,
    0xF3C3CF28, 0x87D843FE, 0x1BF4D684, 0x6FEF5A52, 0x57B670A6, 0x23ADFC70, 0xBF81690A, 0xCB9AE5DC,
    0xCF333CE2, 0xBB28B034, 0x2704254E, 0x531FA998, 0x6B46836C, 0x1F5D0FBA, 0x83719AC0, 0xF76A1616,
    0x8A2228BC, 0xFE39A46A, 0x62153110, 0x160EBDC6, 0x2E579732, 0x5A4C1BE4, 0xC6608E9E, 0xB27B0248,
    0xB6D2DB76, 0xC2C957A0, 0x5EE5C2DA, 0x2AFE4E0C, 0x12A764F8, 0x66BCE82E, 0xFA907D54, 0x8E8BF182,
    0x939C1286, 0xE7879E50, 0x7BAB0B2A, 0x0FB087FC, 0x37E9AD08, 0x43F221DE, 0xDFDEB4A4, 0xABC53872,
    0xAF6CE14C, 0xDB776D9A, 0x475BF8E0, 0x33407436, 0x0B195EC2, 0x7F02D214, 0xE32E476E, 0x9735CBB8,
    0xEA7DF512, 0x9E6679C4, 0x024AECBE, 0x76516068, 0x4E084A9C, 0x3A13C64A, 0xA63F5330, 0xD224DFE6,
    0xD68D06D8, 0xA2968A0E, 0x3EBA1F74, 0x4AA193A2, 0x72F8B956, 0x06E33580, 0x9ACFA0FA, 0xEED42C2C,
    0x605FDDAE, 0x14445178, 0x8868C402, 0xFC7348D4, 0xC42A6220, 0xB031EEF6, 0x2C1D7B8C, 0x5806F75A,
    0x5CAF2E64, 0x28B4A2B2, 0xB49837C8, 0xC083BB1E, 0xF8DA91EA, 0x8CC11D3C, 0x10ED8846, 0x64F60490,
    0x19BE3A3A, 0x6DA5B6EC, 0xF1892396, 0x8592AF40, 0xBDCB85B4, 0xC9D00962, 0x55FC9C18, 0x21E710CE,
    0x254EC9F0, 0x51554526, 0xCD79D05C, 0xB9625C8A, 0x813B767E, 0xF520FAA8, 0x690C6FD2, 0x1D17E304,
    0x5323A9DA, 0x2738250C, 0xBB14B076, 0xCF0F3CA0, 0xF7561654, 0x834D9A82, 0x1F610FF8, 0x6B7A832E,
    0x6FD35A10, 0x1BC8D6C6, 0x87E443BC, 0xF3FFCF6A, 0xCBA6E59E, 0xBFBD6948, 0x2391FC32, 0x578A70E4,
    0x2AC24E4E, 0x5ED9C298, 0xC2F557E2, 0xB6EEDB34, 0x8EB7F1C0, 0xFAAC7D16, 0x6680E86C, 0x129B64BA,
    0x1632BD84, 0x62293152, 0xFE05A428, 0x8A1E28FE, 0xB247020A, 0xC65C8EDC, 0x5A701BA6, 0x2E6B9770,
    0xA0E066F2, 0xD4FBEA24, 0x48D77F5E, 0x3CCCF388, 0x0495D97C, 0x708E55AA, 0xECA2C0D0, 0x98B94C06,
    0x9C109538, 0xE80B19EE, 0x74278C94, 0x003C0042, 0x38652AB6, 0x4C7EA660, 0xD052331A, 0xA449BFCC,
    0xD9018166, 0xAD1A0DB0, 0x313698CA, 0x452D141C, 0x7D743EE8, 0x096FB23E, 0x95432744, 0xE158AB92,
    0xE5F172AC, 0x91EAFE7A, 0x0DC66B00, 0x79DDE7D6, 0x4184CD22, 0x359F41F4, 0xA9B3D48E, 0xDDA85858,
    0xC0BFBB5C, 0xB4A4378A, 0x2888A2F0, 0x5C932E26, 0x64CA04D2, 0x10D18804, 0x8CFD1D7E, 0xF8E691A8,
    0xFC4F4896, 0x8854C440, 0x1478513A, 0x6063DDEC, 0x583AF718, 0x2C217BCE, 0xB00DEEB4, 0xC4166262,
    0xB95E5CC8, 0xCD45D01E, 0x51694564, 0x2572C9B2, 0x1D2BE346, 0x69306F90, 0xF51CFAEA, 0x8107763C,
    0x85AEAF02, 0xF1B523D4, 0x6D99B6AE, 0x19823A78, 0x21DB108C, 0x55C09C5A, 0xC9EC0920, 0xBDF785F6,
    0x337C7474, 0x4767F8A2, 0xDB4B6DD8, 0xAF50E10E, 0x9709CBFA, 0xE312472C, 0x7F3ED256, 0x0B255E80,
    0x0F8C87BE, 0x7B970B68, 0xE7BB9E12, 0x93A012C4, 0xABF93830, 0xDFE2B4E6, 0x43CE219C, 0x37D5AD4A,
    0x4A9D93E0, 0x3E861F36, 0xA2AA8A4C, 0xD6B1069A, 0xEEE82C6E, 0x9AF3A0B8, 0x06DF35C2, 0x72C4B914,
    0x766D602A, 0x0276ECFC, 0x9E5A7986, 0xEA41F550, 0xD218DFA4, 0xA6035372, 0x3A2FC608, 0x4E344ADE
]


def readTextFile(dataset_filename):
    data_list = []
    with open(dataset_filename, 'r') as f:
        lines = f.readlines()
        for line in lines:
            data_list.append(tuple([float(x) for x in line.split(',')[:-1]]))
    return np.array(data_list, dtype=np.ubyte).flatten()


def createSparseTable(dataset_filename, ntype=np.float64):

    try:
        f = open(dataset_filename, 'r')
    except Exception:
        sys.exit("Unable to open '{}'.".format(dataset_filename))

    lines = f.readlines()
    row_index_line = lines[0]
    columns_line = lines[1]
    values_line = lines[2]

    row_offsets = np.array(row_index_line.rstrip('
,').split(','), dtype=np.uint64)
    num_vectors = len(row_offsets) - 1
    col_indices = np.array(columns_line.rstrip('
,').split(','), dtype=np.uint64)
    num_columns = len(col_indices)
    data = np.array(values_line.rstrip('
,').split(','), dtype=ntype)
    num_nonzeros = len(data)

    max_col = 0

    for i in range(num_columns):
        if col_indices[i] > max_col:
            max_col = col_indices[i]
    if sys.version_info[0] < 3:
        num_features = long(max_col)
    else:
        num_features = int(max_col)

    if (
        num_columns != num_nonzeros or
        num_nonzeros != int(row_offsets[num_vectors]) - 1 or
        num_features == 0 or
        num_vectors == 0
    ):
        print("Incorrect format of file")
        sys.exit()

    numeric_table = CSRNumericTable(data, col_indices, row_offsets, num_features, num_vectors)

    return numeric_table


def printAprioriItemsets(large_itemsets_table, large_itemsets_support_table,
                         itemsets_to_print=20):
    large_itemset_count = large_itemsets_support_table.getNumberOfRows()
    num_items_in_large_itemsets = large_itemsets_table.getNumberOfRows()

    large_itemsets = large_itemsets_table.getBlockOfRowsAsDouble(0, num_items_in_large_itemsets).flatten()
    large_itemsets_support_data = large_itemsets_support_table.getBlockOfRowsAsDouble(0, large_itemset_count).flatten()

    large_itemsets_array = [[] for x in range(large_itemset_count)]

    for i in range(num_items_in_large_itemsets):
        large_itemsets_array[int(large_itemsets[2 * i])].append(large_itemsets[2 * i + 1])

    support_array = [0] * large_itemset_count

    for i in range(large_itemset_count):
        support_array[int(large_itemsets_support_data[2 * i])] = large_itemsets_support_data[2 * i + 1]

    print("
Apriori example program results
")
    print("Last " + str(itemsets_to_print) + " large itemsets: 
")
    print("Itemset			Support")

    if large_itemset_count > itemsets_to_print and itemsets_to_print != 0:
        i_min = large_itemset_count - itemsets_to_print
    else:
        i_min = 0

    for i in range(i_min, large_itemset_count):
        print("{", end='')
        for l in range(len(large_itemsets_array[i]) - 1):
            print("{:.0f}".format(large_itemsets_array[i][l]), end=', ')
        print("{:.0f}}}		".format(large_itemsets_array[i][len(large_itemsets_array[i]) - 1]), end='')

        print("{:.0f}".format(support_array[i]))


def printAprioriRules(left_items_table, right_items_table, confidence_table,
                      num_rules_to_print=20):

    num_rules = confidence_table.getNumberOfRows()
    num_left_items = left_items_table.getNumberOfRows()
    num_right_items = right_items_table.getNumberOfRows()

    left_items = left_items_table.getBlockOfRowsAsDouble(0, num_left_items).flatten()
    right_items = right_items_table.getBlockOfRowsAsDouble(0, num_right_items).flatten()
    confidence = confidence_table.getBlockOfRowsAsDouble(0, num_rules).flatten()

    left_items_array = [[] for x in range(num_rules)]

    if num_rules == 0:
        print("
No association rules were found ")
        return

    for i in range(num_left_items):
        left_items_array[int(left_items[2 * i])].append(left_items[2 * i + 1])

    right_items_array = [[] for x in range(num_right_items)]

    for i in range(num_right_items):
        right_items_array[int(right_items[2 * i])].append(right_items[2 * i + 1])

    confidence_array = [0] * num_rules

    for i in range(num_rules):
        confidence_array[i] = confidence[i]

    print("
Last {} association rules: 
".format(num_rules_to_print))
    print("Rule				Confidence")
    if num_rules > num_rules_to_print and num_rules_to_print != 0:
        i_min = num_rules - num_rules_to_print
    else:
        i_min = 0

    for i in range(i_min, num_rules):
        print("{", end='')
        for l in range(len(left_items_array[i]) - 1):
            print("{:.0f}, ".format(left_items_array[i][l]), end='')
        print("{:.0f}}} => {{".format(left_items_array[i][len(left_items_array[i]) - 1]), end='')

        for l in range(len(right_items_array[i]) - 1):
            print("{:.0f}, ".format(right_items_array[i][l]), end='')
        print("{:.0f}}}		".format(right_items_array[i][len(right_items_array[i]) - 1]), end='')

        print("{0:.6g}".format(confidence_array[i]))


def isFull(layout):
    layout_int = int(layout)
    if packed_mask & layout_int:
        return False
    return True


def printArray(array, num_printed_cols, num_printed_rows, num_cols, message,
               interval=10, flt64=True):
    print(message)
    flat_array = array.flatten()
    decimals = '3' if flt64 else '0'
    for i in range(num_printed_rows):
        for j in range(num_printed_cols):
            print("{:<{width}.{dec}f}".format(
                flat_array[i * num_cols + j], width=interval, dec=decimals), end=''
            )
        print()
    print()


def isUpper(layout):
    if (
        layout == NumericTableIface.upperPackedSymmetricMatrix or
        layout == NumericTableIface.upperPackedTriangularMatrix
    ):
        return True
    return False


def isLower(layout):
    if (
        layout == NumericTableIface.lowerPackedSymmetricMatrix or
        layout == NumericTableIface.lowerPackedTriangularMatrix
    ):
        return True
    return False


def printLowerArray(array, num_printed_rows, message, interval=10):
    cols = 1
    print(message)
    for i in range(num_printed_rows):
        for j in range(cols):
            print("{:<{width}.3f}".format(array[i][j], width=interval), end='')
        print()
        cols += 1
    print()


def printUpperArray(array, num_printed_cols, num_printed_rows, num_cols,
                    message, interval=10):
    flat_array = array.flatten()
    print(message)
    ind = 0
    for i in range(num_printed_rows):
        for j in range(i):
            print(' ' * 10)
        for j in range(i, num_printed_cols):
            print(flat_array[ind])
            ind += 1
        for j in range(num_printed_cols, num_cols):
            ind += 1
        print()
    print()


def printNumericTable(data_table, message='', num_printed_rows=0, num_printed_cols=0,
                      interval=10):
    num_rows = data_table.getNumberOfRows()
    num_cols = data_table.getNumberOfColumns()
    layout = data_table.getDataLayout()

    if num_printed_rows != 0:
        num_printed_rows = min(num_rows, num_printed_rows)
    else:
        num_printed_rows = num_rows

    if num_printed_cols != 0:
        num_printed_cols = min(num_cols, num_printed_cols)
    else:
        num_printed_cols = num_cols

    block = BlockDescriptor()
    if isFull(layout) or layout == NumericTableIface.csrArray:
        data_table.getBlockOfRows(0, num_rows, readOnly, block)
        printArray(block.getArray(), num_printed_cols, num_printed_rows,
                   num_cols, message, interval)
        data_table.releaseBlockOfRows(block)
    else:
        packed_table = data_table.getBlockOfRowsAsDouble(0, num_rows)

        if isLower(layout):
            printLowerArray(packed_table, num_printed_rows, message, interval)
        elif isUpper(layout):
            printUpperArray(packed_table, num_printed_cols, num_printed_rows,
                            num_cols, message, interval)


def printNumericTables(data_table_1, data_table_2, title_1='', title_2='',
                       message='', num_printed_rows=0, interval=10, flt64=True):
    num_rows_1 = data_table_1.getNumberOfRows()
    num_rows_2 = data_table_2.getNumberOfRows()
    num_cols_1 = data_table_1.getNumberOfColumns()
    num_cols_2 = data_table_2.getNumberOfColumns()

    num_rows = min(num_rows_1, num_rows_2)
    if num_printed_rows != 0:
        num_rows = min(min(num_rows_1, num_rows_2), num_printed_rows)

    block1 = BlockDescriptor()
    block2 = BlockDescriptor()
    data_table_1.getBlockOfRows(0, num_rows, readOnly, block1)
    data_table_2.getBlockOfRows(0, num_rows, readOnly, block2)

    data_float64_1 = block1.getArray()
    data_float64_2 = block2.getArray()

    data_float64_1 = data_float64_1.flatten()
    data_float64_2 = data_float64_2.flatten()

    decimals = '3' if flt64 else '0'

    print(message)
    print("{:<{width}}".format(title_1, width=(interval * num_cols_1)), end='')
    print("{:<{width}}".format(title_2, width=(interval * num_cols_2)))
    for i in range(num_rows):
        for j in range(num_cols_1):
            print("{:<{width}.{dec}f}".format(data_float64_1[i * num_cols_1 + j], width=interval, dec=decimals), end='')
        for j in range(num_cols_2):
            print("{:<{width}.0f}".format(data_float64_2[i * num_cols_2 + j], width=interval), end='')
        print()
    print()

    data_table_1.releaseBlockOfRows(block1)
    data_table_2.releaseBlockOfRows(block2)


def getCRC32(input, prevRes=0):
    from binascii import crc32
    return crc32(input, prevRes)


def copyBytes(dst, src, size):
    for i in range(size):
        dst[i] = src[i]

def printALSRatings(usersOffsetTable, itemsOffsetTable, ratings):
    nUsers = ratings.getNumberOfRows()
    nItems = ratings.getNumberOfColumns()
    ratingsData = ratings.getBlockOfRowsAsDouble(0, nUsers).flatten()
    usersOffset = usersOffsetTable.getBlockOfRowsAsInt(0, 1).flatten()[0]
    itemsOffset = itemsOffsetTable.getBlockOfRowsAsInt(0, 1).flatten()[0]

    print(" User ID, Item ID, rating")
    for i in range(nUsers):
        for j in range(nItems):
            print("{}, {}, {:.6g}".format(i + usersOffset, j + itemsOffset, ratingsData[i * nItems + j]))



def printTensor(dataTable, message="", nPrintedRows=0, nPrintedCols=0, interval=10):
    dims = dataTable.getDimensions()
    nRows = int(dims[0])

    if nPrintedRows != 0:
        nPrintedRows = min(nRows, nPrintedRows)
    else:
        nPrintedRows = nRows

    block = SubtensorDescriptor()

    dataTable.getSubtensor([], 0, nPrintedRows, readOnly, block)

    nCols = int(block.getSize() / nPrintedRows)

    if nPrintedCols != 0:
        nPrintedCols = min(nCols, nPrintedCols)
    else:
        nPrintedCols = nCols

    printArray(block.getArray(), int(nPrintedCols), int(nPrintedRows), int(nCols), message, interval)
    dataTable.releaseSubtensor(block)


def printTensors(dataTable1, dataTable2, title1="", title2="", message="", nPrintedRows=0, interval=15):
    dims1 = dataTable1.getDimensions()
    nRows1 = int(dims1[0])

    if nPrintedRows != 0:
        nPrintedRows = min(nRows1, nPrintedRows)
    else:
        nPrintedRows = nRows1

    block1 = SubtensorDescriptor()
    dataTable1.getSubtensor([], 0, nPrintedRows, readOnly, block1)
    nCols1 = int(block1.getSize() / nPrintedRows)

    dims2 = dataTable2.getDimensions()
    nRows2 = int(dims2[0])

    if nPrintedRows != 0:
        nPrintedRows = min(nRows2, nPrintedRows)
    else:
        nPrintedRows = nRows2

    block2 = SubtensorDescriptor()
    dataTable2.getSubtensor([], 0, nPrintedRows, readOnly, block2)
    nCols2 = int(block2.getSize() / nPrintedRows)

    dataType1 = block1.getArray().flatten()
    dataType2 = block2.getArray().flatten()

    print(message)
    print("{:<{width}}".format(title1, width=(interval * nCols1)), end='')
    print("{:<{width}}".format(title2, width=(interval * nCols2)))

    for i in range(nPrintedRows):
        for j in range(nCols1):
            print("{v:<{width}.0f}".format(v=dataType1[i * nCols1 + j], width=interval), end='')

        for j in range(nCols2):
            print("{:<{width}.3f}".format(dataType2[i * nCols2 + j], width=int(interval / 2)), end='')
        print()
    print()

    dataTable1.releaseSubtensor(block1)
    dataTable2.releaseSubtensor(block2)


def printTensor3d(dataTable, message="", nFirstDim=0, nSecondDim=0, interval=10):
    dims = dataTable.getDimensions()
    nRows = int(dims[0])
    nCols = int(dims[1])

    if nFirstDim != 0:
        nFirstDim = min(nRows, nFirstDim)
    else:
        nFirstDim = nRows

    if nSecondDim != 0:
        nSecondDim = min(nCols, nSecondDim)
    else:
        nSecondDim = nCols

    block = SubtensorDescriptor()

    print(message)
    for i in range(nFirstDim):
        dataTable.getSubtensor([i], 0, nSecondDim, readOnly, block)

        nThirdDim = block.getSize() / nSecondDim

        printArray(block.getArray(), int(nThirdDim), int(nSecondDim), int(nThirdDim), "", interval)

        dataTable.releaseSubtensor(block)


def readTensorFromCSV(datasetFileName, allowOneColumn=False):
    dataSource = FileDataSource(datasetFileName,
                                DataSourceIface.doAllocateNumericTable,
                                DataSourceIface.doDictionaryFromContext)
    dataSource.loadDataBlock()

    nt = dataSource.getNumericTable()
    size = nt.getNumberOfRows()
    block = BlockDescriptor()
    nt.getBlockOfRows(0, size, readOnly, block)
    blockData = block.getArray().flatten()

    dims = [size]
    if nt.getNumberOfColumns() > 1 or allowOneColumn:
        dims.append(nt.getNumberOfColumns())
        size *= dims[1]

    tensorData = np.array(blockData, dtype=np.float32)

    nt.releaseBlockOfRows(block)

    tensorData.shape = dims
    tensor = HomogenTensor(tensorData, ntype=np.float32)

    return tensor