Custom-Distinct implementation

Qlik

Hashing attributes (columns) that define distinct value for the records (rows)

_some_table:
LOAD
    * // _hash_for_some_attrs comes from there
From [_some_table.qvd];

Concatenate LOAD * Where not Exists(_hash_for_some_attrs);
LOAD
    // other, ...
    , dup_avoid_1
    , dup_avoid_2
    // ..., other
    // -- additions by hereby ETL
    , Hash256(dup_avoid_1, dup_avoid_2) as _hash_for_some_attrs
From [_some_table_datasource];

STORE _some_table into [_some_table.qvd]; DROP Table _some_table;


// -----------------------------------------------
// Another demo-sample

t:
NoConcatenate
LOAD *, Hash256(a, b) as _h;
LOAD * Inline [
a, b, v
1, 1, t-v-1
2, 2, t-v-2
3, 2, t-v-3
];

Concatenate
LOAD * Where not Exists(_h, Hash256(a, b));
LOAD * Inline [
a, b, v
1, 1, x-1
2, 1, x-2
3, 1, x-3
1, 2, x-4
2, 2, x-5
3, 2, x-6
];

DROP Field _h;