D-Net

"""

Examples:

    >>> table1('''

    ... James   10	2

    ... Mark    7	3

    ... Lila    74	1

    ... ''')

    >>> sql("select * from table1")

    a     | b  | c

    --------------

    James | 10 | 2

    Mark  | 7  | 3

    Lila  | 74 | 1

    >>> sql("coutput '../../tests/output' split:2  mode:rcfile select hashmodarchdep(rank,2),* from (select a as name , b as age, c as rank from table1)")

    return_value

    ------------

    1

    >>> sql("unionallrcfiles file:../../tests/output")

    name  | age | rank

    ------------------

    Mark  | 7   | 3

    Lila  | 74  | 1

    James | 10  | 2

    >>> sql("coutput '../../tests/emptyoutput' split:2  mode:rcfile select hashmodarchdep(rank,2),* from (select a as name , b as age, c as rank from table1 limit 0)")

    return_value

    ------------

    1

    >>> sql("unionallrcfiles file:../../tests/emptyoutput")

    >>> sql("coutput '../../tests/outputsp8' split:8  mode:rcfile select hashmodarchdep(rank,8),* from (select a as name , b as age, c as rank from table1)")

    return_value

    ------------

    1

    >>> sql("unionallrcfiles file:../../tests/outputsp8")

    name  | age | rank

    ------------------

    James | 10  | 2

    Mark  | 7   | 3

    Lila  | 74  | 1

"""

import os.path

import sys

from vtout import SourceNtoOne

import functions

import lib.inoutparsing

import os

import marshal

from itertools import izip , imap

import itertools

import cPickle as cPickle

import struct

import gc

import cStringIO as cStringIO

import marshal

import zlib

from collections import OrderedDict

from array import array

import time

import listser

BLOCK_SIZE = 65536000

ZLIB = "zlib"

BZ2 = "bzip"

RCFILE=1

SDC=2

registered=True

def getSize(v):

    t = type(v)

    if t == unicode:

        return 52 + 4*len(v)

    if t in (int, float, None):

        return 24

    return 37 + len(v)

def outputData(diter, schema, connection, *args, **formatArgs):

    ### Parameter handling ###

    where=None

    mode = 'row'

    compression = 'zlib'

    level = 5

    if len(args)>0:

        where=args[0]

    elif 'file' in formatArgs:

        where=formatArgs['file']

    else:

        raise functions.OperatorError(__name__.rsplit('.')[-1],"No destination provided")

    if 'file' in formatArgs:

        del formatArgs['file']

    if 'mode' in formatArgs:

        mode = formatArgs['mode']

    if 'compr'  in formatArgs:

        if formatArgs['compr'] == "zlib":

            compression = ZLIB

        elif formatArgs['compr'] == "bz2":

            compression = BZ2

    if 'level' in formatArgs:

        l = formatArgs['level']

        if int(l)>=0 and int(l) <=9 :

            level = int(l)

    filename, ext=os.path.splitext(os.path.basename(where))

    fullpath=os.path.split(where)[0]

    if 'split' not in formatArgs:

        fileIter=open(where, "w+b")

        fastPickler = cPickle.Pickler(fileIter, 1)

        fastPickler.fast = 1

    else:

        fileIter = 1

    if mode == 'simplecol':

        pass

    if mode == 'spaconedict':

        colnum = len(schema)

        marshal.dump(schema,fileIter,2)

        setcol = set([])

        start = 0

        step = 65535 / colnum

        bsize = 0

        numrows = 0

        dictlimit = step - colnum

        current = [[] for _ in xrange(colnum)]

        prevdicts = {}

        count = 0

        while True:

            maxlen = 0

            nditer = zip(*itertools.islice(diter, 0, step))

            if len(nditer) != 0:

                count += len(nditer[0])

            for i,col in enumerate(nditer):

                current[i] += col

                setcol.update(col)

                l = len(setcol)

                if l > maxlen:

                    maxlen = l

            step = (65535 - maxlen) / colnum

            if step < 5000 or len(nditer) == 0:

                prev = fileIter.tell()

                headindex = [0 for _ in xrange(colnum+2)]

                fileIter.write(struct.pack('L'*len(headindex), *headindex))

                headindex[i] = fileIter.tell()

                s = sorted(setcol)

                finallist =[]

                for val in s:

                    if val not in prevdicts:

                        finallist.append(val)

                prevdicts = dict(((x,y) for y,x in enumerate(s)))

                marshal.dump(list(finallist),fileIter,2)

                cur = zip(*current)

                for i in xrange(count):

                    array('H',[prevdicts[y] for y in cur[i]]).tofile(fileIter)

                headindex[i+1] = fileIter.tell()

                headindex[i+2] = count

                count=0

                fileIter.seek(prev)

                fileIter.write(struct.pack('L'*len(headindex), *headindex))

                fileIter.seek(headindex[colnum])

                current = [[] for _ in xrange(colnum)]

                setcol = set([])

                gc.collect()

                step = 65535 / colnum

            if len(nditer)==0:

                break

    if mode == 'sortedspac':

        colnum = len(schema)

        marshal.dump(schema,fileIter,2)

        setcol = [set([]) for _ in xrange(colnum)]

#        compressorlistcols = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistdicts = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistvals = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

        listoflens = [0 for _ in xrange(colnum)]

        start = 0

        dictsize = 65536

        listofarrays = []

        paxcols = []

        indextype = 'H'

        step = dictsize

        stopstep = int(dictsize/20)

        bsize = 0

        listofnditers = []

        numrows=0

        dictlimit = step - colnum

        coldicts = [{} for _ in xrange(colnum)]

        prevsets =  [set([]) for _ in xrange(colnum)]

        count = 0

        blocknum = 0

        bsize = 0

        while True:

            maxlen = 0

            ret = False

            z = zip(*itertools.islice(diter, 0, step))

            if z==[]:

                ret = True

            else:

                for c in z:

                    bsize+=sys.getsizeof(c)

                listofnditers.append(z)

                if len(listofnditers[-1])!=0:

                    count += len(listofnditers[-1][0])

                else:

                    ret = 1

                for i,col in enumerate(listofnditers[-1]):

                    if i not in paxcols:

                        setcol[i].update(col)

                        l = len(setcol[i])

                        if l > maxlen:

                            maxlen = l

            step = dictsize - maxlen

            if step < stopstep or ret or bsize>300000000:

                bsize=0

                prev = fileIter.tell() + 4*(colnum+2)

                output = cStringIO.StringIO()

                headindex = [0 for _ in xrange(colnum+2)]

                if blocknum == 0:

                    for i in xrange(colnum):

                        headindex[i] = output.tell() + prev

#                        s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

#                        if (dictsize*2+len(marshal.dumps(setcol[i],2))>len(marshal.dumps(s,2))):

#                            paxcols.append(i)

#                            output.write(compressorlistdicts[i].compress(marshal.dumps(s,2)))

                        if (len(setcol[i])>55000):

                            paxcols.append(i)

                            s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

                            #output.write(compressorlistdicts[i].compress(marshal.dumps(s,2)))

                            listofarrays.append(s)

                        else:

                            prevsets[i] = set(setcol[i]).copy()

                            s = sorted(setcol[i])

                            coldicts[i] = dict(((x,y) for y,x in enumerate(s)))

                            coldict = coldicts[i]

                            if len(s)<256:

                                indextype='B'

                            else:

                                indextype='H'

                            listofarrays.append(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]))

                            #output.write(compressorlistcols[i].compress(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl] ).tostring()))

                else:

                    for i in xrange(colnum):

                        headindex[i] = output.tell() + prev

                        if i in paxcols:

                            s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

                            #output.write(compressorlistdicts[i].compress(marshal.dumps(s,2)))

                            listofarrays.append(s)

                        else:

                            difnew = list(setcol[i]-prevsets[i])

                            difold = list(prevsets[i]-setcol[i])

                            prevsets[i].intersection_update(setcol[i])

                            prevsets[i].update(difnew)

                            prevsets[i].update(difold[:(dictsize-len(prevsets[i]))])

                            towrite = {}

                            le = len(difold)

                            si = len(coldicts[i])

                            for l,j in enumerate(difnew):

                                if l<le:

                                    towrite[j] = coldicts[i][difold[l]]

                                else:

                                    towrite[j] = si

                                    si+=1

                            coldicts[i] = dict(((x,y) for y,x in enumerate(prevsets[i])))

                            coldict = coldicts[i]

                            if len(prevsets[i]) != 0 :

                                if len(prevsets[i])<256:

                                    indextype='B'

                                else:

                                    indextype='H'

                                output.write(zlib.compress(marshal.dumps(towrite.keys(),2)))

                                output.write(zlib.compress(array(indextype,towrite.values()).tostring()))

                            listofarrays.append(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]))

                        #output.write(compressorlistcols[i].compress(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl] ).tostring()))

#

                z = zip(*listofarrays)

                ##edw elegxw prwta oti uparxei kapoio paxcol

                z.sort(key=lambda z: z[paxcols[0]])

                li = zip(*z)

#                for i,l in enumerate(listofarrays):

#                    if i in paxcols:

#                        output.write(zlib.compress(marshal.dumps(sorted(l),2)))

#                    else:

#                        output.write(zlib.compress(marshal.dumps(l)))

                for x in li:

                       output.write(zlib.compress(marshal.dumps(sorted(x),2)))

                headindex[i+1] = output.tell()+ prev

                headindex[i+2] = count

                count=0

                fileIter.write(struct.pack('L'*len(headindex), *headindex))

                fileIter.write(output.getvalue())

                listoflens = [0 for _ in xrange(colnum)]

                for s in setcol:

                    s.clear()

                listofarrays = []

                listofnditers = []

                gc.collect()

                step = dictsize

                blocknum+=1

            if ret:

                break

    if mode == 'sspac':

        colnum = len(schema)

        marshal.dump(schema,fileIter,2)

        setcol = [set([]) for _ in xrange(colnum)]

#        compressorlistcols = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistdicts = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistvals = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

        listoflens = [0 for _ in xrange(colnum)]

        start = 0

        dictsize = 65536

        #listofarrays = []

        paxcols = []

        indextype = 'H'

        step = dictsize

        stopstep = int(dictsize/20)

        bsize = 0

        listofnditers = []

        numrows=0

        dictlimit = step - colnum

        coldicts = [{} for _ in xrange(colnum)]

        prevsets =  [set([]) for _ in xrange(colnum)]

        count = 0

        blocknum = 0

        bsize = 0

        while True:

            maxlen = 0

            ret = False

            z = zip(*itertools.islice(diter, 0, step))

            if z==[]:

                ret = True

            else:

                for c in z:

                    bsize+=sys.getsizeof(c)

                listofnditers.append(z)

                if len(listofnditers[-1])!=0:

                    count += len(listofnditers[-1][0])

                else:

                    ret = 1

                for i,col in enumerate(listofnditers[-1]):

                    if i not in paxcols:

                        setcol[i].update(col)

                        l = len(setcol[i])

                        if l > maxlen:

                            maxlen = l

            step = dictsize - maxlen

            if step < stopstep or ret or bsize>300000000:

                bsize=0

                prev = fileIter.tell() + 4*(colnum+2)

                output = cStringIO.StringIO()

                headindex = [0 for _ in xrange(colnum+2)]

                listofvals = []

                for j in xrange(colnum):

                     listofvals.append([val for subl in [x[j] for x in listofnditers] for val in subl])

                listofvals1 = zip(*listofvals)

                listofvals1.sort(key=lambda listofvals1: listofvals1[10])

                listofvals = zip(*listofvals1)

                for s in setcol:

                    s.clear()

                for l in xrange(colnum):

                    setcol[l] = set(listofvals[l])

                print listofvals[10]

                if blocknum == 0:

                    for i in xrange(colnum):

                        headindex[i] = output.tell() + prev

#                        s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

#                        if (dictsize*2+len(marshal.dumps(setcol[i],2))>len(marshal.dumps(s,2))):

#                            paxcols.append(i)

#                            output.write(compressorlistdicts[i].compress(marshal.dumps(s,2)))

                        if (len(setcol[i])>55000):

                            paxcols.append(i)

                            #s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

                            output.write(zlib.compress(marshal.dumps(listofvals[i],2)))

                            #listofarrays.append(s)

                        else:

                            prevsets[i] = set(setcol[i]).copy()

                            s = sorted(setcol[i])

                            coldicts[i] = dict(((x,y) for y,x in enumerate(s)))

                            coldict = coldicts[i]

                            if len(s)<256:

                                indextype='B'

                            else:

                                indextype='H'

                            #listofarrays.append(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]))

                            output.write(zlib.compress(marshal.dumps(s,2)))

                            output.write(zlib.compress(array(indextype,[coldict[val] for val in listofvals[i] ]).tostring()))

                else:

                    for i in xrange(colnum):

                        headindex[i] = output.tell() + prev

                        if i in paxcols:

                            #s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

                            output.write(zlib.compress(marshal.dumps(listofvals[i],2)))

                            #listofarrays.append(s)

                        else:

                            difnew = list(setcol[i]-prevsets[i])

                            difold = list(prevsets[i]-setcol[i])

                            prevsets[i].intersection_update(setcol[i])

                            prevsets[i].update(difnew)

                            prevsets[i].update(difold[:(dictsize-len(prevsets[i]))])

                            towrite = {}

                            le = len(difold)

                            si = len(coldicts[i])

                            for l,j in enumerate(difnew):

                                if l<le:

                                    towrite[j] = coldicts[i][difold[l]]

                                else:

                                    towrite[j] = si

                                    si+=1

                            coldicts[i] = dict(((x,y) for y,x in enumerate(prevsets[i])))

                            coldict = coldicts[i]

                            if len(prevsets[i]) != 0 :

                                if len(prevsets[i])<256:

                                    indextype='B'

                                else:

                                    indextype='H'

                                output.write(zlib.compress(marshal.dumps(towrite.keys(),2)))

                                output.write(zlib.compress(array(indextype,towrite.values()).tostring()))

                            #listofarrays.append(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]))

                            output.write(zlib.compress(array(indextype,[coldict[val] for val in listofvals[i]] ).tostring()))

#                for i,l in enumerate(listofarrays):

#                    if i in paxcols:

#                        output.write(zlib.compress(marshal.dumps(sorted(l),2)))

#                    else:

#                        output.write(zlib.compress(l.tostring()))

                headindex[i+1] = output.tell()+ prev

                headindex[i+2] = count

                count=0

                fileIter.write(struct.pack('L'*len(headindex), *headindex))

                fileIter.write(output.getvalue())

                listoflens = [0 for _ in xrange(colnum)]

                for s in setcol:

                    s.clear()

                #listofarrays = []

                listofnditers = []

                listofvals=[]

                gc.collect()

                step = dictsize

                blocknum+=1

            if ret:

                break

    if mode == 'sortspac':

        colnum = len(schema)

        marshal.dump(schema,fileIter,2)

        setcol = [set([]) for _ in xrange(colnum)]

#        compressorlistcols = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistdicts = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistvals = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

        listoflens = [0 for _ in xrange(colnum)]

        start = 0

        dictsize = 65536

        #listofarrays = []

        paxcols = []

        indextype = 'H'

        step = dictsize

        stopstep = int(dictsize/20)

        bsize = 0

        listofnditers = []

        numrows=0

        dictlimit = step - colnum

        coldicts = [{} for _ in xrange(colnum)]

        prevsets =  [set([]) for _ in xrange(colnum)]

        count = 0

        blocknum = 0

        bsize = 0

        while True:

            maxlen = 0

            ret = False

            z = zip(*itertools.islice(diter, 0, step))

            if z==[]:

                ret = True

            else:

                for c in z:

                    bsize+=sys.getsizeof(c)

                listofnditers.append(z)

                if len(listofnditers[-1])!=0:

                    count += len(listofnditers[-1][0])

                else:

                    ret = 1

                for i,col in enumerate(listofnditers[-1]):

                    if i not in paxcols:

                        setcol[i].update(col)

                        l = len(setcol[i])

                        if l > maxlen:

                            maxlen = l

            step = dictsize - maxlen

            if step < stopstep or ret or bsize>300000000:

                bsize=0

                prev = fileIter.tell() + 4*(colnum+2)

                output = cStringIO.StringIO()

                headindex = [0 for _ in xrange(colnum+2)]

                listofvals = []

                for j in xrange(colnum):

                    listofvals.append([val for subl in [x[j] for x in listofnditers] for val in subl])

                listofvals1 = zip(*listofvals)

                listofvals1.sort(key=lambda listofvals1: listofvals1[5])

                listofvals = zip(*listofvals1)

                for s in setcol:

                    s.clear()

                for l in xrange(colnum):

                    setcol[l] = set(listofvals[l])

                if blocknum == 0:

                    for i in xrange(colnum):

                        headindex[i] = output.tell() + prev

#                        s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

#                        if (dictsize*2+len(marshal.dumps(setcol[i],2))>len(marshal.dumps(s,2))):

#                            paxcols.append(i)

#                            output.write(compressorlistdicts[i].compress(marshal.dumps(s,2)))

                        if (len(setcol[i])>55000):

                            paxcols.append(i)

                            print i

                            output.write(zlib.compress(marshal.dumps(listofvals[i],2)))

                            #listofarrays.append(s)

                        else:

                            prevsets[i] = set(setcol[i]).copy()

                            s = sorted(setcol[i])

                            coldicts[i] = dict(((x,y) for y,x in enumerate(s)))

                            coldict = coldicts[i]

                            if len(s)<256:

                                indextype='B'

                            else:

                                indextype='H'

                            #listofarrays.append(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]))

                            output.write(zlib.compress(marshal.dumps(s,2)))

                            output.write(zlib.compress(array(indextype,[coldict[val] for val in listofvals[i]] ).tostring()))

                            #l = [coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]

                            #z = [coldict[val] for val in listofvals[i]]

                            #print l

                            #print i

                            #print len(z)

                else:

#                    listofnditers1 = zip(*listofnditers)

#                    listofnditers1.sort(key=lambda listofnditers1: listofnditers1[paxcols[0]])

#                    listofnditers = zip(*listofnditers1)

                    for i in xrange(colnum):

                        headindex[i] = output.tell() + prev

                        if i in paxcols:

                            s =  [val for subl in [x[i] for x in listofnditers] for val in subl]

                            output.write(zlib.compress(marshal.dumps(listofvals[i],2)))

                            #listofarrays.append(s)

                        else:

                            difnew = list(setcol[i]-prevsets[i])

                            difold = list(prevsets[i]-setcol[i])

                            prevsets[i].intersection_update(setcol[i])

                            prevsets[i].update(difnew)

                            prevsets[i].update(difold[:(dictsize-len(prevsets[i]))])

                            towrite = {}

                            le = len(difold)

                            si = len(coldicts[i])

                            for l,j in enumerate(difnew):

                                if l<le:

                                    towrite[j] = coldicts[i][difold[l]]

                                else:

                                    towrite[j] = si

                                    si+=1

                            coldicts[i] = dict(((x,y) for y,x in enumerate(prevsets[i])))

                            coldict = coldicts[i]

                            if len(prevsets[i]) != 0 :

                                if len(prevsets[i])<256:

                                    indextype='B'

                                else:

                                    indextype='H'

                                output.write(zlib.compress(marshal.dumps(towrite.keys(),2)))

                                output.write(zlib.compress(array(indextype,towrite.values()).tostring()))

                            #listofarrays.append(array(indextype,[coldict[val] for subl in [x[i] for x in listofnditers] for val in subl]))

                            output.write(zlib.compress(array(indextype,[coldict[val] for val in listofvals[i]] ).tostring()))

#                for i,l in enumerate(listofarrays):

#                    if i in paxcols:

#                        output.write(zlib.compress(marshal.dumps(sorted(l),2)))

#                    else:

#                        output.write(zlib.compress(l.tostring()))

                headindex[i+1] = output.tell()+ prev

                headindex[i+2] = count

                count=0

                fileIter.write(struct.pack('L'*len(headindex), *headindex))

                fileIter.write(output.getvalue())

                listoflens = [0 for _ in xrange(colnum)]

                for s in setcol:

                    s.clear()

                #listofarrays = []

                listofnditers = []

                gc.collect()

                step = dictsize

                blocknum+=1

            if ret:

                break

    if mode == 'corelspac':

        colnum = len(schema)

        marshal.dump(schema,fileIter,2)

        setcol = [set([]) for _ in xrange(colnum)]

#        compressorlistcols = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistdicts = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

#        compressorlistvals = [zlib.BZ2Compressor(9) for _ in xrange(colnum)]

        listoflens = [0 for _ in xrange(colnum)]

        start = 0

        dictsize = 65536

        #listofarrays = []

        paxcols = []

        indextype = 'H'

        step = dictsize

        stopstep = int(dictsize/20)

        bsize = 0

        listofnditers = []

        numrows=0

        dictlimit = step - colnum

        coldicts = [{} for _ in xrange(colnum)]

        prevsets =  [set([]) for _ in xrange(colnum)]

        corelset = {}

        correlatedcols = [{} for _ in xrange(colnum**colnum)]

        correlated = set()

        firstgroup = set()

        count = 0

        blocknum = 0

        bsize = 0

        def allgroup(expansions, n=0, groups = []):

            expgroup = [expansions[n]]

            if n == len(expansions) - 1:

                yield groups + [expgroup]

                for i in xrange(len(groups)):

                    tmp = groups[i]

                    groups[i] = tmp + expgroup

                    yield groups

                    groups[i] = tmp

            else:

                for g in allgroup(expansions, n+1, groups + [expgroup]):

                    yield g

                for i in xrange(len(groups)):

                    tmp = groups[i]

                    groups[i] = tmp + expgroup

                    for g in allgroup(expansions, n + 1, groups):

                        yield g

                    groups[i] = tmp

        for i in allgroup(range(colnum)):

            print i

        while True:

            maxlen = 0

            ret = False

            z = zip(*itertools.islice(diter, 0, step))

            if z==[]:

                ret = True

            else:

                for c in z:

                    bsize+=sys.getsizeof(c)

                listofnditers.append(z)

                if len(listofnditers[-1])!=0:

                    count += len(listofnditers[-1][0])

                else:

                    ret = 1

                for i,col in enumerate(listofnditers[-1]):

                    if i not in paxcols:

                        setcol[i].update(col)

                        l = len(setcol[i])

                        if l > maxlen:

                            maxlen = l

            step = dictsize - maxlen

            if step < stopstep or ret or bsize>300000000:

                bsize=0

                prev = fileIter.tell() + 4*(colnum+2)

                output = cStringIO.StringIO()

                headindex = [0 for _ in xrange(colnum+2)]

                listofvals = []

                for j in xrange(colnum):

                    listofvals.append([val for subl in [x[j] for x in listofnditers] for val in subl])

                if blocknum == 0:

                    sortdict = {}

                    for i in xrange(colnum):

                        if (len(setcol[i])>55000):

                            print i

                            paxcols.append(i)

                        sortdict[i] = len(setcol[i])

#                    le = sorted(sortdict, key=sortdict.get)

#                    ginomeno = 1

#

#                    for i in le:

#                        ginomeno = ginomeno * sortdict[i]

#                        if ginomeno > 256:

#                            break

#                        else:

#                            pass #firstgroup.add(i)

#                    num = colnum-len(paxcols)-len(firstgroup)

#                    if num%2 != 0 and len(firstgroup)!=0:

#                        firstgroup.remove(le[0])

#

#                    for i in xrange(colnum-1):

#                        for j in xrange(i+1,colnum):

#                            if  i not in correlated and j not in correlated and i not in firstgroup and j not in firstgroup: